shivam 0.0.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/test.yml +57 -0
- data/.gitignore +10 -0
- data/.rspec +1 -0
- data/.yardopts +5 -0
- data/CHANGELOG.md +899 -0
- data/Gemfile +35 -0
- data/Guardfile +14 -0
- data/LICENSE +23 -0
- data/README.md +679 -0
- data/Rakefile +21 -0
- data/bin/ci/before_build.sh +20 -0
- data/bin/ci/before_build_docker.sh +20 -0
- data/bin/ci/install_on_debian.sh +46 -0
- data/bin/hutch +8 -0
- data/examples/consumer.rb +13 -0
- data/examples/producer.rb +10 -0
- data/hutch.gemspec +27 -0
- data/lib/hutch/acknowledgements/base.rb +16 -0
- data/lib/hutch/acknowledgements/nack_on_all_failures.rb +19 -0
- data/lib/hutch/adapter.rb +11 -0
- data/lib/hutch/adapters/bunny.rb +37 -0
- data/lib/hutch/adapters/march_hare.rb +41 -0
- data/lib/hutch/broker.rb +384 -0
- data/lib/hutch/cli.rb +246 -0
- data/lib/hutch/config.rb +305 -0
- data/lib/hutch/consumer.rb +125 -0
- data/lib/hutch/error_handlers/airbrake.rb +54 -0
- data/lib/hutch/error_handlers/base.rb +15 -0
- data/lib/hutch/error_handlers/bugsnag.rb +30 -0
- data/lib/hutch/error_handlers/honeybadger.rb +43 -0
- data/lib/hutch/error_handlers/logger.rb +22 -0
- data/lib/hutch/error_handlers/rollbar.rb +28 -0
- data/lib/hutch/error_handlers/sentry.rb +26 -0
- data/lib/hutch/error_handlers/sentry_raven.rb +31 -0
- data/lib/hutch/error_handlers.rb +11 -0
- data/lib/hutch/exceptions.rb +14 -0
- data/lib/hutch/logging.rb +32 -0
- data/lib/hutch/message.rb +31 -0
- data/lib/hutch/publisher.rb +75 -0
- data/lib/hutch/serializers/identity.rb +19 -0
- data/lib/hutch/serializers/json.rb +22 -0
- data/lib/hutch/tracers/datadog.rb +18 -0
- data/lib/hutch/tracers/newrelic.rb +19 -0
- data/lib/hutch/tracers/null_tracer.rb +15 -0
- data/lib/hutch/tracers.rb +7 -0
- data/lib/hutch/version.rb +3 -0
- data/lib/hutch/waiter.rb +104 -0
- data/lib/hutch/worker.rb +145 -0
- data/lib/hutch.rb +69 -0
- data/lib/yard-settings/handler.rb +38 -0
- data/lib/yard-settings/yard-settings.rb +2 -0
- data/spec/hutch/broker_spec.rb +462 -0
- data/spec/hutch/cli_spec.rb +93 -0
- data/spec/hutch/config_spec.rb +259 -0
- data/spec/hutch/consumer_spec.rb +208 -0
- data/spec/hutch/error_handlers/airbrake_spec.rb +49 -0
- data/spec/hutch/error_handlers/bugsnag_spec.rb +55 -0
- data/spec/hutch/error_handlers/honeybadger_spec.rb +58 -0
- data/spec/hutch/error_handlers/logger_spec.rb +28 -0
- data/spec/hutch/error_handlers/rollbar_spec.rb +45 -0
- data/spec/hutch/error_handlers/sentry_raven_spec.rb +37 -0
- data/spec/hutch/error_handlers/sentry_spec.rb +47 -0
- data/spec/hutch/logger_spec.rb +34 -0
- data/spec/hutch/message_spec.rb +38 -0
- data/spec/hutch/serializers/json_spec.rb +17 -0
- data/spec/hutch/tracers/datadog_spec.rb +44 -0
- data/spec/hutch/waiter_spec.rb +51 -0
- data/spec/hutch/worker_spec.rb +184 -0
- data/spec/hutch_spec.rb +87 -0
- data/spec/spec_helper.rb +42 -0
- data/templates/default/class/html/settings.erb +0 -0
- data/templates/default/class/setup.rb +4 -0
- data/templates/default/fulldoc/html/css/hutch.css +13 -0
- data/templates/default/layout/html/setup.rb +7 -0
- data/templates/default/method_details/html/settings.erb +5 -0
- data/templates/default/method_details/setup.rb +4 -0
- data/templates/default/method_details/text/settings.erb +0 -0
- data/templates/default/module/html/settings.erb +40 -0
- data/templates/default/module/setup.rb +4 -0
- metadata +205 -0
data/lib/hutch/worker.rb
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
require 'hutch/message'
|
2
|
+
require 'hutch/logging'
|
3
|
+
require 'hutch/broker'
|
4
|
+
require 'hutch/acknowledgements/nack_on_all_failures'
|
5
|
+
require 'hutch/waiter'
|
6
|
+
require 'carrot-top'
|
7
|
+
require 'securerandom'
|
8
|
+
|
9
|
+
module Hutch
|
10
|
+
class Worker
|
11
|
+
include Logging
|
12
|
+
|
13
|
+
def initialize(broker, consumers, setup_procs)
|
14
|
+
@broker = broker
|
15
|
+
self.consumers = consumers
|
16
|
+
self.setup_procs = setup_procs
|
17
|
+
end
|
18
|
+
|
19
|
+
# Run the main event loop. The consumers will be set up with queues, and
|
20
|
+
# process the messages in their respective queues indefinitely. This method
|
21
|
+
# never returns.
|
22
|
+
def run
|
23
|
+
setup_queues
|
24
|
+
setup_procs.each(&:call)
|
25
|
+
|
26
|
+
Waiter.wait_until_signaled
|
27
|
+
|
28
|
+
stop
|
29
|
+
end
|
30
|
+
|
31
|
+
# Stop a running worker by killing all subscriber threads.
|
32
|
+
def stop
|
33
|
+
@broker.stop
|
34
|
+
end
|
35
|
+
|
36
|
+
# Set up the queues for each of the worker's consumers.
|
37
|
+
def setup_queues
|
38
|
+
logger.info 'setting up queues'
|
39
|
+
vetted = @consumers.reject { |c| group_configured? && group_restricted?(c) }
|
40
|
+
vetted.each do |c|
|
41
|
+
setup_queue(c)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# Bind a consumer's routing keys to its queue, and set up a subscription to
|
46
|
+
# receive messages sent to the queue.
|
47
|
+
def setup_queue(consumer)
|
48
|
+
logger.info "setting up queue: #{consumer.get_queue_name}"
|
49
|
+
|
50
|
+
queue = @broker.queue(consumer.get_queue_name, consumer.get_options)
|
51
|
+
@broker.bind_queue(queue, consumer.routing_keys)
|
52
|
+
|
53
|
+
queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
|
54
|
+
delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
|
55
|
+
handle_message(consumer, delivery_info, properties, payload)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# Called internally when a new messages comes in from RabbitMQ. Responsible
|
60
|
+
# for wrapping up the message and passing it to the consumer.
|
61
|
+
def handle_message(consumer, delivery_info, properties, payload)
|
62
|
+
serializer = consumer.get_serializer || Hutch::Config[:serializer]
|
63
|
+
logger.debug {
|
64
|
+
spec = serializer.binary? ? "#{payload.bytesize} bytes" : "#{payload}"
|
65
|
+
"message(#{properties.message_id || '-'}): " +
|
66
|
+
"routing key: #{delivery_info.routing_key}, " +
|
67
|
+
"consumer: #{consumer}, " +
|
68
|
+
"payload: #{spec}"
|
69
|
+
}
|
70
|
+
|
71
|
+
message = Message.new(delivery_info, properties, payload, serializer)
|
72
|
+
consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
|
73
|
+
with_tracing(consumer_instance).handle(message)
|
74
|
+
@broker.ack(delivery_info.delivery_tag) unless consumer_instance.message_rejected?
|
75
|
+
rescue => ex
|
76
|
+
acknowledge_error(delivery_info, properties, @broker, ex)
|
77
|
+
handle_error(properties, payload, consumer, ex)
|
78
|
+
end
|
79
|
+
|
80
|
+
def with_tracing(klass)
|
81
|
+
Hutch::Config[:tracer].new(klass)
|
82
|
+
end
|
83
|
+
|
84
|
+
def handle_error(*args)
|
85
|
+
Hutch::Config[:error_handlers].each do |backend|
|
86
|
+
backend.handle(*args)
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def acknowledge_error(delivery_info, properties, broker, ex)
|
91
|
+
acks = error_acknowledgements +
|
92
|
+
[Hutch::Acknowledgements::NackOnAllFailures.new]
|
93
|
+
acks.find do |backend|
|
94
|
+
backend.handle(delivery_info, properties, broker, ex)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def consumers=(val)
|
99
|
+
if val.empty?
|
100
|
+
logger.warn "no consumer loaded, ensure there's no configuration issue"
|
101
|
+
end
|
102
|
+
@consumers = val
|
103
|
+
end
|
104
|
+
|
105
|
+
def error_acknowledgements
|
106
|
+
Hutch::Config[:error_acknowledgements]
|
107
|
+
end
|
108
|
+
|
109
|
+
private
|
110
|
+
|
111
|
+
def group_configured?
|
112
|
+
if group.present? && consumer_groups.blank?
|
113
|
+
logger.info 'Consumer groups are blank'
|
114
|
+
end
|
115
|
+
group.present?
|
116
|
+
end
|
117
|
+
|
118
|
+
def group_restricted?(consumer)
|
119
|
+
consumers_to_load = consumer_groups[group]
|
120
|
+
if consumers_to_load
|
121
|
+
!consumers_to_load.include?(consumer.name)
|
122
|
+
else
|
123
|
+
true
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def group
|
128
|
+
Hutch::Config[:group]
|
129
|
+
end
|
130
|
+
|
131
|
+
def consumer_groups
|
132
|
+
Hutch::Config[:consumer_groups]
|
133
|
+
end
|
134
|
+
|
135
|
+
attr_accessor :setup_procs
|
136
|
+
|
137
|
+
def unique_consumer_tag
|
138
|
+
prefix = Hutch::Config[:consumer_tag_prefix]
|
139
|
+
unique_part = SecureRandom.uuid
|
140
|
+
"#{prefix}-#{unique_part}".tap do |tag|
|
141
|
+
raise "Tag must be 255 bytes long at most, current one is #{tag.bytesize} ('#{tag}')" if tag.bytesize > 255
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
data/lib/hutch.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'hutch/adapter'
|
2
|
+
require 'hutch/consumer'
|
3
|
+
require 'hutch/worker'
|
4
|
+
require 'hutch/broker'
|
5
|
+
require 'hutch/logging'
|
6
|
+
require 'hutch/serializers/identity'
|
7
|
+
require 'hutch/serializers/json'
|
8
|
+
require 'hutch/config'
|
9
|
+
require 'hutch/message'
|
10
|
+
require 'hutch/cli'
|
11
|
+
require 'hutch/version'
|
12
|
+
require 'hutch/error_handlers'
|
13
|
+
require 'hutch/exceptions'
|
14
|
+
require 'hutch/tracers'
|
15
|
+
|
16
|
+
module Hutch
|
17
|
+
@@connection_mutex = Mutex.new
|
18
|
+
|
19
|
+
def self.register_consumer(consumer)
|
20
|
+
self.consumers << consumer
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.consumers
|
24
|
+
@consumers ||= []
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.logger
|
28
|
+
Hutch::Logging.logger
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.global_properties=(properties)
|
32
|
+
@global_properties = properties
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.global_properties
|
36
|
+
@global_properties ||= {}
|
37
|
+
end
|
38
|
+
|
39
|
+
# Connects to broker, if not yet connected.
|
40
|
+
#
|
41
|
+
# @param options [Hash] Connection options
|
42
|
+
# @param config [Hash] Configuration
|
43
|
+
# @option options [Boolean] :enable_http_api_use
|
44
|
+
def self.connect(options = {}, config = Hutch::Config)
|
45
|
+
@@connection_mutex.synchronize do
|
46
|
+
unless connected?
|
47
|
+
@broker = Hutch::Broker.new(config)
|
48
|
+
@broker.connect(options)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.disconnect
|
54
|
+
@broker.disconnect if @broker
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.broker
|
58
|
+
@broker
|
59
|
+
end
|
60
|
+
|
61
|
+
# @return [Boolean]
|
62
|
+
def self.connected?
|
63
|
+
broker && broker.connection && broker.connection.open?
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.publish(*args)
|
67
|
+
broker.publish(*args)
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# :nodoc:
|
2
|
+
class SettingsHandlerBase < YARD::Handlers::Ruby::Base
|
3
|
+
handles method_call :string_setting
|
4
|
+
handles method_call :number_setting
|
5
|
+
handles method_call :boolean_setting
|
6
|
+
|
7
|
+
namespace_only
|
8
|
+
|
9
|
+
def process
|
10
|
+
name = statement.parameters.first.jump(:tstring_content, :ident).source
|
11
|
+
object = YARD::CodeObjects::MethodObject.new(namespace, name)
|
12
|
+
register(object)
|
13
|
+
|
14
|
+
# Modify the code object for the new instance method
|
15
|
+
object.dynamic = true
|
16
|
+
# Add custom metadata to the object
|
17
|
+
object['custom_field'] = '(Found using method_missing)'
|
18
|
+
|
19
|
+
# Module-level configuration notes
|
20
|
+
hutch_config = YARD::CodeObjects::ModuleObject.new(:root, "Hutch::Config")
|
21
|
+
collection_name = statement.first.first
|
22
|
+
default_value = statement.parameters[1].jump(:tstring_content, :ident).source
|
23
|
+
|
24
|
+
(hutch_config['setting_rows'] ||= []) << {
|
25
|
+
name: name,
|
26
|
+
default_value: default_value,
|
27
|
+
type: collection_name.sub('_setting', '').capitalize,
|
28
|
+
description: object.docstring,
|
29
|
+
first_line_of_description: first_line_of_description(object)
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
def first_line_of_description(object)
|
34
|
+
return '' if object.docstring.blank?
|
35
|
+
|
36
|
+
object.docstring.lines.first
|
37
|
+
end
|
38
|
+
end
|