rabbitek 0.3.5 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/Gemfile.lock +12 -7
- data/README.md +1 -1
- data/lib/rabbitek.rb +13 -0
- data/lib/rabbitek/cli.rb +4 -1
- data/lib/rabbitek/server/batcher.rb +1 -1
- data/lib/rabbitek/server/consumer.rb +4 -4
- data/lib/rabbitek/server/hooks/retry.rb +1 -1
- data/lib/rabbitek/server/hooks/time_tracker.rb +25 -4
- data/lib/rabbitek/server/message_processor.rb +84 -0
- data/lib/rabbitek/server/starter.rb +15 -51
- data/lib/rabbitek/version.rb +1 -1
- data/rabbitek.gemspec +2 -1
- metadata +25 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d6e7e9681a2b0440a71d3ab9ae5f15e614034ab51c6c0339905c5130733e42ad
|
4
|
+
data.tar.gz: a6b4d63873a74d3026d576aa9ef6d438e67c79e8ab47cf3634572c62da93ff8d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 67cf264c46c092a763e4d0b48b895305d69d915c68489a54d4bd3b220c6f4ce611147afa83d00097ac71b2870390770ebcd5b27c4338fb7550e85ee458af3bd9
|
7
|
+
data.tar.gz: d976bcbb478c7c90e964c4f32cf3852a529415e6a3c08414459863a9e4a434dc900734cdeddc7495824853ab4a76e0b02d97566cd2f7325b8ab0ee60bc13da59
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
rabbitek (0.
|
4
|
+
rabbitek (0.8.0)
|
5
5
|
activesupport (> 3.0)
|
6
6
|
bunny (~> 2.11.0)
|
7
|
-
oj (
|
7
|
+
oj (> 2.0.0)
|
8
8
|
opentracing (~> 0.4)
|
9
9
|
slop (~> 4.0)
|
10
|
+
yabeda (>= 0.6.0)
|
10
11
|
|
11
12
|
GEM
|
12
13
|
remote: https://rubygems.org/
|
@@ -52,16 +53,17 @@ GEM
|
|
52
53
|
i18n (>= 0.7, < 2)
|
53
54
|
minitest (~> 5.1)
|
54
55
|
tzinfo (~> 1.1)
|
55
|
-
amq-protocol (2.3.
|
56
|
+
amq-protocol (2.3.2)
|
56
57
|
arel (9.0.0)
|
57
58
|
ast (2.4.0)
|
58
59
|
builder (3.2.4)
|
59
60
|
bunny (2.11.0)
|
60
61
|
amq-protocol (~> 2.3.0)
|
61
62
|
coderay (1.1.2)
|
62
|
-
concurrent-ruby (1.1.
|
63
|
+
concurrent-ruby (1.1.6)
|
63
64
|
crass (1.0.6)
|
64
65
|
diff-lcs (1.3)
|
66
|
+
dry-initializer (3.0.3)
|
65
67
|
erubi (1.9.0)
|
66
68
|
globalid (0.4.2)
|
67
69
|
activesupport (>= 4.2.0)
|
@@ -83,7 +85,7 @@ GEM
|
|
83
85
|
nio4r (2.5.2)
|
84
86
|
nokogiri (1.10.7)
|
85
87
|
mini_portile2 (~> 2.4.0)
|
86
|
-
oj (3.10.
|
88
|
+
oj (3.10.8)
|
87
89
|
opentracing (0.5.0)
|
88
90
|
parallel (1.12.1)
|
89
91
|
parser (2.5.1.2)
|
@@ -143,7 +145,7 @@ GEM
|
|
143
145
|
ruby-progressbar (~> 1.7)
|
144
146
|
unicode-display_width (~> 1.0, >= 1.0.1)
|
145
147
|
ruby-progressbar (1.10.0)
|
146
|
-
slop (4.
|
148
|
+
slop (4.8.2)
|
147
149
|
sprockets (4.0.0)
|
148
150
|
concurrent-ruby (~> 1.0)
|
149
151
|
rack (> 1, < 3)
|
@@ -159,6 +161,9 @@ GEM
|
|
159
161
|
websocket-driver (0.7.1)
|
160
162
|
websocket-extensions (>= 0.1.0)
|
161
163
|
websocket-extensions (0.1.4)
|
164
|
+
yabeda (0.6.1)
|
165
|
+
concurrent-ruby
|
166
|
+
dry-initializer
|
162
167
|
|
163
168
|
PLATFORMS
|
164
169
|
ruby
|
@@ -173,4 +178,4 @@ DEPENDENCIES
|
|
173
178
|
rubocop (~> 0.58.0)
|
174
179
|
|
175
180
|
BUNDLED WITH
|
176
|
-
2.
|
181
|
+
2.1.4
|
data/README.md
CHANGED
@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
|
|
11
11
|
* OpenTracing (http://opentracing.io/) instrumentation
|
12
12
|
* NewRelic instrumentation for sending errors
|
13
13
|
* Sentry instrumentation for sending errors
|
14
|
+
* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
|
14
15
|
|
15
16
|
## Installation
|
16
17
|
|
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
|
|
89
90
|
* dead queue
|
90
91
|
* CRON jobs
|
91
92
|
* extended docs and how to
|
92
|
-
* prometheus metrics
|
93
93
|
|
94
94
|
|
95
95
|
## Development
|
data/lib/rabbitek.rb
CHANGED
@@ -6,6 +6,7 @@ require 'bunny'
|
|
6
6
|
require 'oj'
|
7
7
|
require 'opentracing'
|
8
8
|
require 'logger'
|
9
|
+
require 'yabeda'
|
9
10
|
|
10
11
|
# active_support
|
11
12
|
require 'active_support/core_ext/module/attribute_accessors'
|
@@ -54,3 +55,15 @@ module Rabbitek
|
|
54
55
|
@bunny_connection ||= BunnyConnection.initialize_connection
|
55
56
|
end
|
56
57
|
end
|
58
|
+
|
59
|
+
Yabeda.configure do
|
60
|
+
group :rabbitek do
|
61
|
+
counter :processed_messages_count, comment: 'Total number of all messages'
|
62
|
+
counter :errored_messages_count, comment: 'Total number of errored messages'
|
63
|
+
histogram :processed_messages_runtime do
|
64
|
+
comment 'How long it takes to process message'
|
65
|
+
unit :seconds
|
66
|
+
buckets [0.1, 0.5, 1, 5, 10, 30, 60]
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/rabbitek/cli.rb
CHANGED
@@ -14,9 +14,12 @@ module Rabbitek
|
|
14
14
|
class CLI
|
15
15
|
include ::Rabbitek::Loggable
|
16
16
|
|
17
|
-
def run
|
17
|
+
def run # rubocop:disable Metrics/AbcSize
|
18
18
|
opts
|
19
19
|
require_application
|
20
|
+
|
21
|
+
Yabeda.configure! unless Yabeda.already_configured?
|
22
|
+
|
20
23
|
map_consumer_workers!
|
21
24
|
|
22
25
|
start_log
|
@@ -52,16 +52,16 @@ module Rabbitek
|
|
52
52
|
Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
|
53
53
|
end
|
54
54
|
|
55
|
-
def
|
56
|
-
self.class.
|
55
|
+
def opts
|
56
|
+
self.class.opts
|
57
57
|
end
|
58
58
|
|
59
59
|
module ClassMethods # rubocop:disable Style/Documentation
|
60
|
-
attr_accessor :rabbit_options_hash, :
|
60
|
+
attr_accessor :rabbit_options_hash, :opts
|
61
61
|
|
62
62
|
def rabbit_options(opts)
|
63
63
|
self.rabbit_options_hash = default_rabbit_options(opts).with_indifferent_access.merge(opts)
|
64
|
-
self.
|
64
|
+
self.opts = opts
|
65
65
|
end
|
66
66
|
|
67
67
|
def perform_async(payload, opts: {}, channel: nil)
|
@@ -11,19 +11,40 @@ module Rabbitek
|
|
11
11
|
include Loggable
|
12
12
|
|
13
13
|
def call(consumer, message)
|
14
|
-
|
14
|
+
log_started(consumer, message)
|
15
15
|
|
16
|
-
start =
|
16
|
+
start = current_time
|
17
17
|
|
18
18
|
super
|
19
19
|
ensure
|
20
|
+
total_time = current_time - start
|
21
|
+
|
22
|
+
log_finished(consumer, message, total_time)
|
23
|
+
metrics_measure_time(consumer, total_time)
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def current_time
|
29
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
30
|
+
end
|
31
|
+
|
32
|
+
def log_started(consumer, message)
|
33
|
+
info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
|
34
|
+
end
|
35
|
+
|
36
|
+
def log_finished(consumer, message, total_time)
|
20
37
|
info(
|
21
|
-
message: 'Finished',
|
38
|
+
message: 'Finished job',
|
22
39
|
consumer: message.delivery_info.routing_key,
|
23
|
-
time:
|
40
|
+
time: total_time,
|
24
41
|
jid: consumer.jid
|
25
42
|
)
|
26
43
|
end
|
44
|
+
|
45
|
+
def metrics_measure_time(consumer, total_time)
|
46
|
+
Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
|
47
|
+
end
|
27
48
|
end
|
28
49
|
end
|
29
50
|
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'forwardable'
|
4
|
+
|
5
|
+
module Rabbitek
|
6
|
+
##
|
7
|
+
# Single message processor
|
8
|
+
class MessageProcessor
|
9
|
+
include Loggable
|
10
|
+
extend Forwardable
|
11
|
+
|
12
|
+
def initialize(starter, delivery_info, properties, payload)
|
13
|
+
@starter = starter
|
14
|
+
@delivery_info = delivery_info
|
15
|
+
@properties = properties
|
16
|
+
@payload = payload
|
17
|
+
end
|
18
|
+
|
19
|
+
def process
|
20
|
+
consumer.set_context
|
21
|
+
|
22
|
+
metrics_add_processed_count
|
23
|
+
|
24
|
+
hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
|
25
|
+
hook_walker.call!(consumer, message) { |*args| run_job(*args) }
|
26
|
+
rescue StandardError => e
|
27
|
+
on_message_errored(e)
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
attr_reader :starter, :delivery_info, :properties, :payload
|
33
|
+
def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
|
34
|
+
|
35
|
+
def run_job(modified_consumer, message)
|
36
|
+
if modified_consumer.opts[:batch]
|
37
|
+
run_job_batched(modified_consumer, message)
|
38
|
+
else
|
39
|
+
modified_consumer.perform(message)
|
40
|
+
modified_consumer.ack!(message.delivery_info) unless modified_consumer.opts[:manual_ack]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def run_job_batched(modified_consumer, message)
|
45
|
+
Batcher.new(modified_consumer).perform(message) do |batch|
|
46
|
+
modified_consumer.perform(batch)
|
47
|
+
modified_consumer.ack!(batch.last.delivery_info, true) unless modified_consumer.opts[:manual_ack]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def on_message_errored(exception)
|
52
|
+
error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
|
53
|
+
metrics_add_errored_count
|
54
|
+
end
|
55
|
+
|
56
|
+
def metrics_add_processed_count
|
57
|
+
Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
|
58
|
+
end
|
59
|
+
|
60
|
+
def metrics_add_errored_count
|
61
|
+
Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
|
62
|
+
end
|
63
|
+
|
64
|
+
def message
|
65
|
+
@message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
|
66
|
+
end
|
67
|
+
|
68
|
+
def consumer
|
69
|
+
@consumer ||= consumer_instance(message.delivery_info.routing_key)
|
70
|
+
end
|
71
|
+
|
72
|
+
def consumer_instance(routing_key)
|
73
|
+
Thread.current[:worker_classes] ||= {}
|
74
|
+
klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
|
75
|
+
klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
|
76
|
+
rescue NameError
|
77
|
+
nil # TODO: to dead queue
|
78
|
+
end
|
79
|
+
|
80
|
+
def metrics_tags
|
81
|
+
{ consumer: consumer.class }
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -17,58 +17,18 @@ module Rabbitek
|
|
17
17
|
setup_bindings!
|
18
18
|
|
19
19
|
work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
|
20
|
-
|
21
|
-
|
20
|
+
Rabbitek.reloader.call do
|
21
|
+
MessageProcessor.new(self, delivery_info, properties, payload).process
|
22
|
+
end
|
22
23
|
end
|
23
24
|
end
|
24
25
|
|
25
|
-
private
|
26
|
-
|
27
|
-
attr_reader :connection, :queue_name, :consumers, :opts
|
28
|
-
|
29
|
-
def setup_bindings!
|
30
|
-
consumers.each do |worker_class|
|
31
|
-
work_queue.bind(work_exchange, routing_key: worker_class.to_s)
|
32
|
-
retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
def on_message_received(message)
|
37
|
-
consumer = consumer_instance(message.delivery_info.routing_key)
|
38
|
-
consumer.set_context
|
39
|
-
|
40
|
-
hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
|
41
|
-
|
42
|
-
hook_walker.call!(consumer, message) do |*args|
|
43
|
-
run_job(*args)
|
44
|
-
end
|
45
|
-
rescue StandardError => e
|
46
|
-
error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
|
47
|
-
end
|
48
|
-
|
49
|
-
def run_job(consumer, message)
|
50
|
-
if consumer.class.batch
|
51
|
-
run_job_batched(consumer, message)
|
52
|
-
else
|
53
|
-
consumer.perform(message)
|
54
|
-
consumer.ack!(message.delivery_info)
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def consumer_instance(routing_key)
|
59
|
-
Thread.current[:worker_classes] ||= {}
|
60
|
-
klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
|
61
|
-
klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
|
62
|
-
rescue NameError
|
63
|
-
nil # TODO: to dead queue
|
64
|
-
end
|
65
|
-
|
66
26
|
def channel
|
67
27
|
@channel ||= begin
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
28
|
+
channel = connection.create_channel
|
29
|
+
channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
|
30
|
+
channel
|
31
|
+
end
|
72
32
|
end
|
73
33
|
|
74
34
|
def work_exchange
|
@@ -95,10 +55,14 @@ module Rabbitek
|
|
95
55
|
)
|
96
56
|
end
|
97
57
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
58
|
+
private
|
59
|
+
|
60
|
+
attr_reader :connection, :queue_name, :consumers, :opts
|
61
|
+
|
62
|
+
def setup_bindings!
|
63
|
+
consumers.each do |worker_class|
|
64
|
+
work_queue.bind(work_exchange, routing_key: worker_class.to_s)
|
65
|
+
retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
|
102
66
|
end
|
103
67
|
end
|
104
68
|
end
|
data/lib/rabbitek/version.rb
CHANGED
data/rabbitek.gemspec
CHANGED
@@ -31,9 +31,10 @@ Gem::Specification.new do |spec|
|
|
31
31
|
|
32
32
|
spec.add_dependency 'activesupport', '> 3.0'
|
33
33
|
spec.add_dependency 'bunny', '~> 2.11.0'
|
34
|
-
spec.add_dependency 'oj', '
|
34
|
+
spec.add_dependency 'oj', '> 2.0.0'
|
35
35
|
spec.add_dependency 'opentracing', '~> 0.4'
|
36
36
|
spec.add_dependency 'slop', '~> 4.0'
|
37
|
+
spec.add_dependency 'yabeda', '>= 0.6.0'
|
37
38
|
|
38
39
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
39
40
|
spec.add_development_dependency 'pry'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rabbitek
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Boostcom
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -42,16 +42,16 @@ dependencies:
|
|
42
42
|
name: oj
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: 2.0.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: 2.0.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: opentracing
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '4.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: yabeda
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 0.6.0
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 0.6.0
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: bundler
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -204,6 +218,7 @@ files:
|
|
204
218
|
- lib/rabbitek/server/hooks/retry.rb
|
205
219
|
- lib/rabbitek/server/hooks/time_tracker.rb
|
206
220
|
- lib/rabbitek/server/message.rb
|
221
|
+
- lib/rabbitek/server/message_processor.rb
|
207
222
|
- lib/rabbitek/server/retryer.rb
|
208
223
|
- lib/rabbitek/server/server_hook.rb
|
209
224
|
- lib/rabbitek/server/starter.rb
|
@@ -221,7 +236,7 @@ metadata:
|
|
221
236
|
source_code_uri: https://github.com/Boostcom/rabbitek
|
222
237
|
changelog_uri: https://github.com/Boostcom/rabbitek/blob/master/CHANGELOG.md
|
223
238
|
bug_tracker_uri: https://github.com/Boostcom/rabbitek/issues
|
224
|
-
post_install_message:
|
239
|
+
post_install_message:
|
225
240
|
rdoc_options: []
|
226
241
|
require_paths:
|
227
242
|
- lib
|
@@ -236,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
236
251
|
- !ruby/object:Gem::Version
|
237
252
|
version: '0'
|
238
253
|
requirements: []
|
239
|
-
rubygems_version: 3.
|
240
|
-
signing_key:
|
254
|
+
rubygems_version: 3.1.2
|
255
|
+
signing_key:
|
241
256
|
specification_version: 4
|
242
257
|
summary: High performance background job processing
|
243
258
|
test_files: []
|