hutch 0.19.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +7 -0
  3. data/.travis.yml +11 -0
  4. data/CHANGELOG.md +438 -0
  5. data/Gemfile +22 -0
  6. data/Guardfile +5 -0
  7. data/LICENSE +22 -0
  8. data/README.md +317 -0
  9. data/Rakefile +14 -0
  10. data/bin/hutch +8 -0
  11. data/circle.yml +3 -0
  12. data/examples/consumer.rb +13 -0
  13. data/examples/producer.rb +10 -0
  14. data/hutch.gemspec +30 -0
  15. data/lib/hutch.rb +62 -0
  16. data/lib/hutch/adapter.rb +11 -0
  17. data/lib/hutch/adapters/bunny.rb +33 -0
  18. data/lib/hutch/adapters/march_hare.rb +37 -0
  19. data/lib/hutch/broker.rb +374 -0
  20. data/lib/hutch/cli.rb +205 -0
  21. data/lib/hutch/config.rb +125 -0
  22. data/lib/hutch/consumer.rb +75 -0
  23. data/lib/hutch/error_handlers.rb +8 -0
  24. data/lib/hutch/error_handlers/airbrake.rb +26 -0
  25. data/lib/hutch/error_handlers/honeybadger.rb +28 -0
  26. data/lib/hutch/error_handlers/logger.rb +16 -0
  27. data/lib/hutch/error_handlers/sentry.rb +23 -0
  28. data/lib/hutch/exceptions.rb +7 -0
  29. data/lib/hutch/logging.rb +32 -0
  30. data/lib/hutch/message.rb +31 -0
  31. data/lib/hutch/serializers/identity.rb +19 -0
  32. data/lib/hutch/serializers/json.rb +22 -0
  33. data/lib/hutch/tracers.rb +6 -0
  34. data/lib/hutch/tracers/newrelic.rb +19 -0
  35. data/lib/hutch/tracers/null_tracer.rb +15 -0
  36. data/lib/hutch/version.rb +4 -0
  37. data/lib/hutch/worker.rb +143 -0
  38. data/spec/hutch/broker_spec.rb +377 -0
  39. data/spec/hutch/cli_spec.rb +80 -0
  40. data/spec/hutch/config_spec.rb +126 -0
  41. data/spec/hutch/consumer_spec.rb +130 -0
  42. data/spec/hutch/error_handlers/airbrake_spec.rb +34 -0
  43. data/spec/hutch/error_handlers/honeybadger_spec.rb +36 -0
  44. data/spec/hutch/error_handlers/logger_spec.rb +15 -0
  45. data/spec/hutch/error_handlers/sentry_spec.rb +20 -0
  46. data/spec/hutch/logger_spec.rb +28 -0
  47. data/spec/hutch/message_spec.rb +38 -0
  48. data/spec/hutch/serializers/json_spec.rb +17 -0
  49. data/spec/hutch/worker_spec.rb +99 -0
  50. data/spec/hutch_spec.rb +87 -0
  51. data/spec/spec_helper.rb +40 -0
  52. metadata +194 -0
data/lib/hutch/cli.rb ADDED
@@ -0,0 +1,205 @@
1
+ require 'optparse'
2
+
3
+ require 'hutch/version'
4
+ require 'hutch/logging'
5
+ require 'hutch/exceptions'
6
+ require 'hutch/config'
7
+
8
+ module Hutch
9
+ class CLI
10
+ include Logging
11
+
12
+ # Run a Hutch worker with the command line interface.
13
+ def run(argv = ARGV)
14
+ parse_options(argv)
15
+
16
+ ::Process.daemon(true) if Hutch::Config.daemonise
17
+
18
+ write_pid if Hutch::Config.pidfile
19
+
20
+ Hutch.logger.info "hutch booted with pid #{::Process.pid}"
21
+
22
+ if load_app && start_work_loop == :success
23
+ # If we got here, the worker was shut down nicely
24
+ Hutch.logger.info 'hutch shut down gracefully'
25
+ exit 0
26
+ else
27
+ Hutch.logger.info 'hutch terminated due to an error'
28
+ exit 1
29
+ end
30
+ end
31
+
32
+ def load_app
33
+ # Try to load a Rails app in the current directory
34
+ load_rails_app('.') if Hutch::Config.autoload_rails
35
+ Hutch::Config.require_paths.each do |path|
36
+ # See if each path is a Rails app. If so, try to load it.
37
+ next if load_rails_app(path)
38
+
39
+ # Given path is not a Rails app, try requiring it as a file
40
+ logger.info "requiring '#{path}'"
41
+ begin
42
+ # Need to add '.' to load path for relative requires
43
+ $LOAD_PATH << '.'
44
+ require path
45
+ rescue LoadError
46
+ logger.fatal "could not load file '#{path}'"
47
+ return false
48
+ ensure
49
+ # Clean up load path
50
+ $LOAD_PATH.pop
51
+ end
52
+ end
53
+
54
+ # Because of the order things are required when we run the Hutch binary
55
+ # in hutch/bin, the Sentry Raven gem gets required **after** the error
56
+ # handlers are set up. Due to this, we never got any Sentry notifications
57
+ # when an error occurred in any of the consumers.
58
+ if defined?(Raven)
59
+ Hutch::Config[:error_handlers] << Hutch::ErrorHandlers::Sentry.new
60
+ end
61
+
62
+ true
63
+ end
64
+
65
+ def load_rails_app(path)
66
+ # path should point to the app's top level directory
67
+ if File.directory?(path)
68
+ # Smells like a Rails app if it's got a config/environment.rb file
69
+ rails_path = File.expand_path(File.join(path, 'config/environment.rb'))
70
+ if File.exists?(rails_path)
71
+ logger.info "found rails project (#{path}), booting app"
72
+ ENV['RACK_ENV'] ||= ENV['RAILS_ENV'] || 'development'
73
+ require rails_path
74
+ ::Rails.application.eager_load!
75
+ return true
76
+ end
77
+ end
78
+ false
79
+ end
80
+
81
+ # Kick off the work loop. This method returns when the worker is shut down
82
+ # gracefully (with a SIGQUIT, SIGTERM or SIGINT).
83
+ def start_work_loop
84
+ Hutch.connect
85
+ @worker = Hutch::Worker.new(Hutch.broker, Hutch.consumers)
86
+ @worker.run
87
+ :success
88
+ rescue ConnectionError, AuthenticationError, WorkerSetupError => ex
89
+ logger.fatal ex.message
90
+ :error
91
+ end
92
+
93
+ def parse_options(args = ARGV)
94
+ OptionParser.new do |opts|
95
+ opts.banner = 'usage: hutch [options]'
96
+
97
+ opts.on('--mq-host HOST', 'Set the RabbitMQ host') do |host|
98
+ Hutch::Config.mq_host = host
99
+ end
100
+
101
+ opts.on('--mq-port PORT', 'Set the RabbitMQ port') do |port|
102
+ Hutch::Config.mq_port = port
103
+ end
104
+
105
+ opts.on("-t", "--[no-]mq-tls", 'Use TLS for the AMQP connection') do |tls|
106
+ Hutch::Config.mq_tls = tls
107
+ end
108
+
109
+ opts.on('--mq-tls-cert FILE', 'Certificate for TLS client verification') do |file|
110
+ abort "Certificate file '#{file}' not found" unless File.exists?(file)
111
+ Hutch::Config.mq_tls_cert = file
112
+ end
113
+
114
+ opts.on('--mq-tls-key FILE', 'Private key for TLS client verification') do |file|
115
+ abort "Private key file '#{file}' not found" unless File.exists?(file)
116
+ Hutch::Config.mq_tls_key = file
117
+ end
118
+
119
+ opts.on('--mq-exchange EXCHANGE',
120
+ 'Set the RabbitMQ exchange') do |exchange|
121
+ Hutch::Config.mq_exchange = exchange
122
+ end
123
+
124
+ opts.on('--mq-vhost VHOST', 'Set the RabbitMQ vhost') do |vhost|
125
+ Hutch::Config.mq_vhost = vhost
126
+ end
127
+
128
+ opts.on('--mq-username USERNAME',
129
+ 'Set the RabbitMQ username') do |username|
130
+ Hutch::Config.mq_username = username
131
+ end
132
+
133
+ opts.on('--mq-password PASSWORD',
134
+ 'Set the RabbitMQ password') do |password|
135
+ Hutch::Config.mq_password = password
136
+ end
137
+
138
+ opts.on('--mq-api-host HOST', 'Set the RabbitMQ API host') do |host|
139
+ Hutch::Config.mq_api_host = host
140
+ end
141
+
142
+ opts.on('--mq-api-port PORT', 'Set the RabbitMQ API port') do |port|
143
+ Hutch::Config.mq_api_port = port
144
+ end
145
+
146
+ opts.on("-s", "--[no-]mq-api-ssl", 'Use SSL for the RabbitMQ API') do |api_ssl|
147
+ Hutch::Config.mq_api_ssl = api_ssl
148
+ end
149
+
150
+ opts.on('--config FILE', 'Load Hutch configuration from a file') do |file|
151
+ begin
152
+ File.open(file) { |fp| Hutch::Config.load_from_file(fp) }
153
+ rescue Errno::ENOENT
154
+ abort "Config file '#{file}' not found"
155
+ end
156
+ end
157
+
158
+ opts.on('--require PATH', 'Require a Rails app or path') do |path|
159
+ Hutch::Config.require_paths << path
160
+ end
161
+
162
+ opts.on('--[no-]autoload-rails', 'Require the current rails app directory') do |autoload_rails|
163
+ Hutch::Config.autoload_rails = autoload_rails
164
+ end
165
+
166
+ opts.on('-q', '--quiet', 'Quiet logging') do
167
+ Hutch::Config.log_level = Logger::WARN
168
+ end
169
+
170
+ opts.on('-v', '--verbose', 'Verbose logging') do
171
+ Hutch::Config.log_level = Logger::DEBUG
172
+ end
173
+
174
+ opts.on('--namespace NAMESPACE', 'Queue namespace') do |namespace|
175
+ Hutch::Config.namespace = namespace
176
+ end
177
+
178
+ opts.on('-d', '--daemonise', 'Daemonise') do |daemonise|
179
+ Hutch::Config.daemonise = daemonise
180
+ end
181
+
182
+ opts.on('--pidfile PIDFILE', 'Pidfile') do |pidfile|
183
+ Hutch::Config.pidfile = pidfile
184
+ end
185
+
186
+ opts.on('--version', 'Print the version and exit') do
187
+ puts "hutch v#{VERSION}"
188
+ exit 0
189
+ end
190
+
191
+ opts.on('-h', '--help', 'Show this message and exit') do
192
+ puts opts
193
+ exit 0
194
+ end
195
+ end.parse!(args)
196
+ end
197
+
198
+ def write_pid
199
+ pidfile = File.expand_path(Hutch::Config.pidfile)
200
+ Hutch.logger.info "writing pid in #{pidfile}"
201
+ File.open(pidfile, 'w') { |f| f.puts ::Process.pid }
202
+ end
203
+
204
+ end
205
+ end
@@ -0,0 +1,125 @@
1
+ require 'hutch/error_handlers/logger'
2
+ require 'erb'
3
+ require 'logger'
4
+
5
+ module Hutch
6
+ class UnknownAttributeError < StandardError; end
7
+
8
+ module Config
9
+ require 'yaml'
10
+
11
+ def self.initialize(params={})
12
+ @config = {
13
+ mq_host: 'localhost',
14
+ mq_port: 5672,
15
+ mq_exchange: 'hutch', # TODO: should this be required?
16
+ mq_vhost: '/',
17
+ mq_tls: false,
18
+ mq_tls_cert: nil,
19
+ mq_tls_key: nil,
20
+ mq_tls_ca_certificates: nil,
21
+ mq_verify_peer: true,
22
+ mq_username: 'guest',
23
+ mq_password: 'guest',
24
+ mq_api_host: 'localhost',
25
+ mq_api_port: 15672,
26
+ mq_api_ssl: false,
27
+ heartbeat: 30,
28
+ # placeholder, allows specifying connection parameters
29
+ # as a URI.
30
+ uri: nil,
31
+ log_level: Logger::INFO,
32
+ require_paths: [],
33
+ autoload_rails: true,
34
+ error_handlers: [Hutch::ErrorHandlers::Logger.new],
35
+ tracer: Hutch::Tracers::NullTracer,
36
+ namespace: nil,
37
+ daemonise: false,
38
+ pidfile: nil,
39
+ channel_prefetch: 0,
40
+ # enables publisher confirms, leaves it up to the app
41
+ # how they are tracked
42
+ publisher_confirms: false,
43
+ # like `publisher_confirms` above but also
44
+ # forces waiting for a confirm for every publish
45
+ force_publisher_confirms: false,
46
+ # Heroku needs > 10. MK.
47
+ connection_timeout: 11,
48
+ read_timeout: 11,
49
+ write_timeout: 11,
50
+ enable_http_api_use: true,
51
+ # Number of seconds that a running consumer is given
52
+ # to finish its job when gracefully exiting Hutch, before
53
+ # it's killed.
54
+ graceful_exit_timeout: 11,
55
+ client_logger: nil,
56
+
57
+ consumer_pool_size: 1,
58
+
59
+ serializer: Hutch::Serializers::JSON,
60
+ }.merge(params)
61
+ end
62
+
63
+ def self.get(attr)
64
+ check_attr(attr)
65
+ user_config[attr]
66
+ end
67
+
68
+ def self.set(attr, value)
69
+ check_attr(attr)
70
+ user_config[attr] = value
71
+ end
72
+
73
+ class << self
74
+ alias_method :[], :get
75
+ alias_method :[]=, :set
76
+ end
77
+
78
+ def self.check_attr(attr)
79
+ unless user_config.key?(attr)
80
+ raise UnknownAttributeError, "#{attr} is not a valid config attribute"
81
+ end
82
+ end
83
+
84
+ def self.user_config
85
+ initialize unless @config
86
+ @config
87
+ end
88
+
89
+ def self.to_hash
90
+ self.user_config
91
+ end
92
+
93
+ def self.load_from_file(file)
94
+ YAML.load(ERB.new(File.read(file)).result).each do |attr, value|
95
+ Hutch::Config.send("#{attr}=", convert_value(attr, value))
96
+ end
97
+ end
98
+
99
+ def self.convert_value(attr, value)
100
+ case attr
101
+ when "tracer"
102
+ Kernel.const_get(value)
103
+ else
104
+ value
105
+ end
106
+ end
107
+
108
+ def self.method_missing(method, *args, &block)
109
+ attr = method.to_s.sub(/=$/, '').to_sym
110
+ return super unless user_config.key?(attr)
111
+
112
+ if method =~ /=$/
113
+ set(attr, args.first)
114
+ else
115
+ get(attr)
116
+ end
117
+ end
118
+
119
+ private
120
+
121
+ def deep_copy(obj)
122
+ Marshal.load(Marshal.dump(obj))
123
+ end
124
+ end
125
+ end
@@ -0,0 +1,75 @@
1
+ require 'set'
2
+
3
+ module Hutch
4
+ # Include this module in a class to register it as a consumer. Consumers
5
+ # gain a class method called `consume`, which should be used to register
6
+ # the routing keys a consumer is interested in.
7
+ module Consumer
8
+ attr_accessor :broker, :delivery_info
9
+
10
+ def self.included(base)
11
+ base.extend(ClassMethods)
12
+ Hutch.register_consumer(base)
13
+ end
14
+
15
+ def reject!
16
+ broker.reject(delivery_info.delivery_tag)
17
+ end
18
+
19
+ def requeue!
20
+ broker.requeue(delivery_info.delivery_tag)
21
+ end
22
+
23
+ def logger
24
+ Hutch::Logging.logger
25
+ end
26
+
27
+ module ClassMethods
28
+ # Add one or more routing keys to the set of routing keys the consumer
29
+ # wants to subscribe to.
30
+ def consume(*routing_keys)
31
+ @routing_keys = self.routing_keys.union(routing_keys)
32
+ end
33
+
34
+ # Explicitly set the queue name
35
+ def queue_name(name)
36
+ @queue_name = name
37
+ end
38
+
39
+ # Allow to specify custom arguments that will be passed when creating the queue.
40
+ def arguments(arguments = {})
41
+ @arguments = arguments
42
+ end
43
+
44
+ # Set custom serializer class, override global value
45
+ def serializer(name)
46
+ @serializer = name
47
+ end
48
+
49
+ # The RabbitMQ queue name for the consumer. This is derived from the
50
+ # fully-qualified class name. Module separators are replaced with single
51
+ # colons, camelcased class names are converted to snake case.
52
+ def get_queue_name
53
+ return @queue_name unless @queue_name.nil?
54
+ queue_name = self.name.gsub(/::/, ':')
55
+ queue_name.gsub!(/([^A-Z:])([A-Z])/) { "#{$1}_#{$2}" }
56
+ queue_name.downcase
57
+ end
58
+
59
+ # Returns consumer custom arguments.
60
+ def get_arguments
61
+ @arguments || {}
62
+ end
63
+
64
+ # Accessor for the consumer's routing key.
65
+ def routing_keys
66
+ @routing_keys ||= Set.new
67
+ end
68
+
69
+ def get_serializer
70
+ @serializer
71
+ end
72
+ end
73
+ end
74
+ end
75
+
@@ -0,0 +1,8 @@
1
+ module Hutch
2
+ module ErrorHandlers
3
+ autoload :Logger, 'hutch/error_handlers/logger'
4
+ autoload :Sentry, 'hutch/error_handlers/sentry'
5
+ autoload :Honeybadger, 'hutch/error_handlers/honeybadger'
6
+ autoload :Airbrake, 'hutch/error_handlers/airbrake'
7
+ end
8
+ end
@@ -0,0 +1,26 @@
1
+ require 'hutch/logging'
2
+ require 'airbrake'
3
+
4
+ module Hutch
5
+ module ErrorHandlers
6
+ class Airbrake
7
+ include Logging
8
+
9
+ def handle(message_id, payload, consumer, ex)
10
+ prefix = "message(#{message_id || '-'}): "
11
+ logger.error prefix + "Logging event to Airbrake"
12
+ logger.error prefix + "#{ex.class} - #{ex.message}"
13
+ ::Airbrake.notify_or_ignore(ex, {
14
+ :error_class => ex.class.name,
15
+ :error_message => "#{ ex.class.name }: #{ ex.message }",
16
+ :backtrace => ex.backtrace,
17
+ :parameters => {
18
+ :payload => payload,
19
+ :consumer => consumer,
20
+ },
21
+ :cgi_data => ENV.to_hash,
22
+ })
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,28 @@
1
+ require 'hutch/logging'
2
+ require 'honeybadger'
3
+
4
+ module Hutch
5
+ module ErrorHandlers
6
+ class Honeybadger
7
+ include Logging
8
+
9
+ def handle(message_id, payload, consumer, ex)
10
+ prefix = "message(#{message_id || '-'}): "
11
+ logger.error prefix + "Logging event to Honeybadger"
12
+ logger.error prefix + "#{ex.class} - #{ex.message}"
13
+ ::Honeybadger.notify_or_ignore(
14
+ :error_class => ex.class.name,
15
+ :error_message => "#{ ex.class.name }: #{ ex.message }",
16
+ :backtrace => ex.backtrace,
17
+ :context => {
18
+ :message_id => message_id,
19
+ :consumer => consumer
20
+ },
21
+ :parameters => {
22
+ :payload => payload
23
+ }
24
+ )
25
+ end
26
+ end
27
+ end
28
+ end