rabbitek 0.3.5 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +7 -2
- data/README.md +1 -1
- data/lib/rabbitek.rb +13 -0
- data/lib/rabbitek/server/hooks/time_tracker.rb +25 -4
- data/lib/rabbitek/server/message_processor.rb +84 -0
- data/lib/rabbitek/server/starter.rb +15 -51
- data/lib/rabbitek/version.rb +1 -1
- data/rabbitek.gemspec +1 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1f7cc70de57fb560439fe5e93b30b76e419d8e00dcea7379c6b3fe4aa4a6325
|
4
|
+
data.tar.gz: 1eb6bd702c62d2e9cd44f97211cfbe6769c58db9f032073bf0361a137d5c16e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 398811e481a4b197ddaea89a52448d76e158576156aacffb5c3801d0b3122bfb3df1770e15c348500fd5fe59a172b3bdb3d8f0c52e4642a9306a7d983a8c616f
|
7
|
+
data.tar.gz: fcc347a951e4530f723e8d754a58b111b7463cc175a3944249296db049b070898b1b44e509633c7af2c8d633c6fa2fb8c45908c3e09e6fa87c677ef327db8c5c
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rabbitek (0.
|
4
|
+
rabbitek (0.4.0)
|
5
5
|
activesupport (> 3.0)
|
6
6
|
bunny (~> 2.11.0)
|
7
7
|
oj (~> 3.6)
|
8
8
|
opentracing (~> 0.4)
|
9
9
|
slop (~> 4.0)
|
10
|
+
yabeda
|
10
11
|
|
11
12
|
GEM
|
12
13
|
remote: https://rubygems.org/
|
@@ -62,6 +63,7 @@ GEM
|
|
62
63
|
concurrent-ruby (1.1.5)
|
63
64
|
crass (1.0.6)
|
64
65
|
diff-lcs (1.3)
|
66
|
+
dry-initializer (3.0.3)
|
65
67
|
erubi (1.9.0)
|
66
68
|
globalid (0.4.2)
|
67
69
|
activesupport (>= 4.2.0)
|
@@ -83,7 +85,7 @@ GEM
|
|
83
85
|
nio4r (2.5.2)
|
84
86
|
nokogiri (1.10.7)
|
85
87
|
mini_portile2 (~> 2.4.0)
|
86
|
-
oj (3.10.
|
88
|
+
oj (3.10.0)
|
87
89
|
opentracing (0.5.0)
|
88
90
|
parallel (1.12.1)
|
89
91
|
parser (2.5.1.2)
|
@@ -159,6 +161,9 @@ GEM
|
|
159
161
|
websocket-driver (0.7.1)
|
160
162
|
websocket-extensions (>= 0.1.0)
|
161
163
|
websocket-extensions (0.1.4)
|
164
|
+
yabeda (0.3.0)
|
165
|
+
concurrent-ruby
|
166
|
+
dry-initializer
|
162
167
|
|
163
168
|
PLATFORMS
|
164
169
|
ruby
|
data/README.md
CHANGED
@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
|
|
11
11
|
* OpenTracing (http://opentracing.io/) instrumentation
|
12
12
|
* NewRelic instrumentation for sending errors
|
13
13
|
* Sentry instrumentation for sending errors
|
14
|
+
* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
|
14
15
|
|
15
16
|
## Installation
|
16
17
|
|
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
|
|
89
90
|
* dead queue
|
90
91
|
* CRON jobs
|
91
92
|
* extended docs and how to
|
92
|
-
* prometheus metrics
|
93
93
|
|
94
94
|
|
95
95
|
## Development
|
data/lib/rabbitek.rb
CHANGED
@@ -6,6 +6,7 @@ require 'bunny'
|
|
6
6
|
require 'oj'
|
7
7
|
require 'opentracing'
|
8
8
|
require 'logger'
|
9
|
+
require 'yabeda'
|
9
10
|
|
10
11
|
# active_support
|
11
12
|
require 'active_support/core_ext/module/attribute_accessors'
|
@@ -54,3 +55,15 @@ module Rabbitek
|
|
54
55
|
@bunny_connection ||= BunnyConnection.initialize_connection
|
55
56
|
end
|
56
57
|
end
|
58
|
+
|
59
|
+
Yabeda.configure do
|
60
|
+
group :rabbitek do
|
61
|
+
counter :processed_messages_count, comment: 'Total number of all messages'
|
62
|
+
counter :errored_messages_count, comment: 'Total number of errored messages'
|
63
|
+
histogram :processed_messages_runtime do
|
64
|
+
comment 'How long it takes to process message'
|
65
|
+
unit :seconds
|
66
|
+
buckets [0.1, 0.5, 1, 5, 10, 30, 60]
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -11,19 +11,40 @@ module Rabbitek
|
|
11
11
|
include Loggable
|
12
12
|
|
13
13
|
def call(consumer, message)
|
14
|
-
|
14
|
+
log_started(consumer, message)
|
15
15
|
|
16
|
-
start =
|
16
|
+
start = current_time
|
17
17
|
|
18
18
|
super
|
19
19
|
ensure
|
20
|
+
total_time = current_time - start
|
21
|
+
|
22
|
+
log_finished(consumer, message, total_time)
|
23
|
+
metrics_measure_time(consumer, total_time)
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def current_time
|
29
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
30
|
+
end
|
31
|
+
|
32
|
+
def log_started(consumer, message)
|
33
|
+
info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_finished(consumer, message, total_time)
|
20
37
|
info(
|
21
|
-
message: 'Finished',
|
38
|
+
message: 'Finished job',
|
22
39
|
consumer: message.delivery_info.routing_key,
|
23
|
-
time:
|
40
|
+
time: total_time,
|
24
41
|
jid: consumer.jid
|
25
42
|
)
|
26
43
|
end
|
44
|
+
|
45
|
+
def metrics_measure_time(consumer, total_time)
|
46
|
+
Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
|
47
|
+
end
|
27
48
|
end
|
28
49
|
end
|
29
50
|
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
module Rabbitek
|
6
|
+
##
|
7
|
+
# Single message processor
|
8
|
+
class MessageProcessor
|
9
|
+
include Loggable
|
10
|
+
extend Forwardable
|
11
|
+
|
12
|
+
def initialize(starter, delivery_info, properties, payload)
|
13
|
+
@starter = starter
|
14
|
+
@delivery_info = delivery_info
|
15
|
+
@properties = properties
|
16
|
+
@payload = payload
|
17
|
+
end
|
18
|
+
|
19
|
+
def process
|
20
|
+
consumer.set_context
|
21
|
+
|
22
|
+
metrics_add_processed_count
|
23
|
+
|
24
|
+
hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
|
25
|
+
hook_walker.call!(consumer, message) { |*args| run_job(*args) }
|
26
|
+
rescue StandardError => e
|
27
|
+
on_message_errored(e)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :starter, :delivery_info, :properties, :payload
|
33
|
+
def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
|
34
|
+
|
35
|
+
def run_job(modified_consumer, message)
|
36
|
+
if modified_consumer.class.batch
|
37
|
+
run_job_batched(modified_consumer, message)
|
38
|
+
else
|
39
|
+
modified_consumer.perform(message)
|
40
|
+
modified_consumer.ack!(message.delivery_info)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def run_job_batched(modified_consumer, message)
|
45
|
+
Batcher.new(modified_consumer).perform(message) do |batch|
|
46
|
+
modified_consumer.perform(batch)
|
47
|
+
modified_consumer.ack!(batch.last.delivery_info, true)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def on_message_errored(exception)
|
52
|
+
error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
|
53
|
+
metrics_add_errored_count
|
54
|
+
end
|
55
|
+
|
56
|
+
def metrics_add_processed_count
|
57
|
+
Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
|
58
|
+
end
|
59
|
+
|
60
|
+
def metrics_add_errored_count
|
61
|
+
Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
|
62
|
+
end
|
63
|
+
|
64
|
+
def message
|
65
|
+
@message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
|
66
|
+
end
|
67
|
+
|
68
|
+
def consumer
|
69
|
+
@consumer ||= consumer_instance(message.delivery_info.routing_key)
|
70
|
+
end
|
71
|
+
|
72
|
+
def consumer_instance(routing_key)
|
73
|
+
Thread.current[:worker_classes] ||= {}
|
74
|
+
klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
|
75
|
+
klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
|
76
|
+
rescue NameError
|
77
|
+
nil # TODO: to dead queue
|
78
|
+
end
|
79
|
+
|
80
|
+
def metrics_tags
|
81
|
+
{ consumer: consumer.class }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -17,58 +17,18 @@ module Rabbitek
|
|
17
17
|
setup_bindings!
|
18
18
|
|
19
19
|
work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
|
20
|
-
|
21
|
-
|
20
|
+
Rabbitek.reloader.call do
|
21
|
+
MessageProcessor.new(self, delivery_info, properties, payload).process
|
22
|
+
end
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
25
|
-
private
|
26
|
-
|
27
|
-
attr_reader :connection, :queue_name, :consumers, :opts
|
28
|
-
|
29
|
-
def setup_bindings!
|
30
|
-
consumers.each do |worker_class|
|
31
|
-
work_queue.bind(work_exchange, routing_key: worker_class.to_s)
|
32
|
-
retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def on_message_received(message)
|
37
|
-
consumer = consumer_instance(message.delivery_info.routing_key)
|
38
|
-
consumer.set_context
|
39
|
-
|
40
|
-
hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
|
41
|
-
|
42
|
-
hook_walker.call!(consumer, message) do |*args|
|
43
|
-
run_job(*args)
|
44
|
-
end
|
45
|
-
rescue StandardError => e
|
46
|
-
error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
|
47
|
-
end
|
48
|
-
|
49
|
-
def run_job(consumer, message)
|
50
|
-
if consumer.class.batch
|
51
|
-
run_job_batched(consumer, message)
|
52
|
-
else
|
53
|
-
consumer.perform(message)
|
54
|
-
consumer.ack!(message.delivery_info)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def consumer_instance(routing_key)
|
59
|
-
Thread.current[:worker_classes] ||= {}
|
60
|
-
klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
|
61
|
-
klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
|
62
|
-
rescue NameError
|
63
|
-
nil # TODO: to dead queue
|
64
|
-
end
|
65
|
-
|
66
26
|
def channel
|
67
27
|
@channel ||= begin
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
28
|
+
channel = connection.create_channel
|
29
|
+
channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
|
30
|
+
channel
|
31
|
+
end
|
72
32
|
end
|
73
33
|
|
74
34
|
def work_exchange
|
@@ -95,10 +55,14 @@ module Rabbitek
|
|
95
55
|
)
|
96
56
|
end
|
97
57
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
58
|
+
private
|
59
|
+
|
60
|
+
attr_reader :connection, :queue_name, :consumers, :opts
|
61
|
+
|
62
|
+
def setup_bindings!
|
63
|
+
consumers.each do |worker_class|
|
64
|
+
work_queue.bind(work_exchange, routing_key: worker_class.to_s)
|
65
|
+
retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
|
102
66
|
end
|
103
67
|
end
|
104
68
|
end
|
data/lib/rabbitek/version.rb
CHANGED
data/rabbitek.gemspec
CHANGED
@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_dependency 'oj', '~> 3.6'
|
35
35
|
spec.add_dependency 'opentracing', '~> 0.4'
|
36
36
|
spec.add_dependency 'slop', '~> 4.0'
|
37
|
+
spec.add_dependency 'yabeda'
|
37
38
|
|
38
39
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
39
40
|
spec.add_development_dependency 'pry'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rabbitek
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Boostcom
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-01-
|
11
|
+
date: 2020-01-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '4.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: yabeda
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: bundler
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -204,6 +218,7 @@ files:
|
|
204
218
|
- lib/rabbitek/server/hooks/retry.rb
|
205
219
|
- lib/rabbitek/server/hooks/time_tracker.rb
|
206
220
|
- lib/rabbitek/server/message.rb
|
221
|
+
- lib/rabbitek/server/message_processor.rb
|
207
222
|
- lib/rabbitek/server/retryer.rb
|
208
223
|
- lib/rabbitek/server/server_hook.rb
|
209
224
|
- lib/rabbitek/server/starter.rb
|