rabbitek 0.3.5 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a36ba46180d4c5e3c28d3d212ff2174c49ef00618f6d49c7defd49ec0768d5ae
4
- data.tar.gz: 25167637efc19c059e73ff8796a19aef890e961ec3c7c891b2b9aba1429d1ef3
3
+ metadata.gz: d1f7cc70de57fb560439fe5e93b30b76e419d8e00dcea7379c6b3fe4aa4a6325
4
+ data.tar.gz: 1eb6bd702c62d2e9cd44f97211cfbe6769c58db9f032073bf0361a137d5c16e9
5
5
  SHA512:
6
- metadata.gz: 33d4cfcad4e769c15e556b93ba40e9c779aa16ab739a889bd2ec01728b1c84cae25096a559aaaf687875f792ad43789612903c9c6c3b4e222b5a3ba0d801df68
7
- data.tar.gz: 22b95c1863b7d22d66c4b9f2eb02a18c734450ac09b3aa999dd6b5f316fa40e45f7f52b101e8924a1c2f2a1e2b8601ba6830b99c4948659825983927c393f421
6
+ metadata.gz: 398811e481a4b197ddaea89a52448d76e158576156aacffb5c3801d0b3122bfb3df1770e15c348500fd5fe59a172b3bdb3d8f0c52e4642a9306a7d983a8c616f
7
+ data.tar.gz: fcc347a951e4530f723e8d754a58b111b7463cc175a3944249296db049b070898b1b44e509633c7af2c8d633c6fa2fb8c45908c3e09e6fa87c677ef327db8c5c
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## v0.4.0
2
+
3
+ Improvements:
4
+ * Add metrics (universal, you can use e.g. prometheus)
5
+
1
6
  ## v0.3.5
2
7
 
3
8
  Bugfix:
data/Gemfile.lock CHANGED
@@ -1,12 +1,13 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- rabbitek (0.3.4)
4
+ rabbitek (0.4.0)
5
5
  activesupport (> 3.0)
6
6
  bunny (~> 2.11.0)
7
7
  oj (~> 3.6)
8
8
  opentracing (~> 0.4)
9
9
  slop (~> 4.0)
10
+ yabeda
10
11
 
11
12
  GEM
12
13
  remote: https://rubygems.org/
@@ -62,6 +63,7 @@ GEM
62
63
  concurrent-ruby (1.1.5)
63
64
  crass (1.0.6)
64
65
  diff-lcs (1.3)
66
+ dry-initializer (3.0.3)
65
67
  erubi (1.9.0)
66
68
  globalid (0.4.2)
67
69
  activesupport (>= 4.2.0)
@@ -83,7 +85,7 @@ GEM
83
85
  nio4r (2.5.2)
84
86
  nokogiri (1.10.7)
85
87
  mini_portile2 (~> 2.4.0)
86
- oj (3.10.1)
88
+ oj (3.10.0)
87
89
  opentracing (0.5.0)
88
90
  parallel (1.12.1)
89
91
  parser (2.5.1.2)
@@ -159,6 +161,9 @@ GEM
159
161
  websocket-driver (0.7.1)
160
162
  websocket-extensions (>= 0.1.0)
161
163
  websocket-extensions (0.1.4)
164
+ yabeda (0.3.0)
165
+ concurrent-ruby
166
+ dry-initializer
162
167
 
163
168
  PLATFORMS
164
169
  ruby
data/README.md CHANGED
@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
11
11
  * OpenTracing (http://opentracing.io/) instrumentation
12
12
  * NewRelic instrumentation for sending errors
13
13
  * Sentry instrumentation for sending errors
14
+ * Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
14
15
 
15
16
  ## Installation
16
17
 
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
89
90
  * dead queue
90
91
  * CRON jobs
91
92
  * extended docs and how to
92
- * prometheus metrics
93
93
 
94
94
 
95
95
  ## Development
data/lib/rabbitek.rb CHANGED
@@ -6,6 +6,7 @@ require 'bunny'
6
6
  require 'oj'
7
7
  require 'opentracing'
8
8
  require 'logger'
9
+ require 'yabeda'
9
10
 
10
11
  # active_support
11
12
  require 'active_support/core_ext/module/attribute_accessors'
@@ -54,3 +55,15 @@ module Rabbitek
54
55
  @bunny_connection ||= BunnyConnection.initialize_connection
55
56
  end
56
57
  end
58
+
59
+ Yabeda.configure do
60
+ group :rabbitek do
61
+ counter :processed_messages_count, comment: 'Total number of all messages'
62
+ counter :errored_messages_count, comment: 'Total number of errored messages'
63
+ histogram :processed_messages_runtime do
64
+ comment 'How long it takes to process message'
65
+ unit :seconds
66
+ buckets [0.1, 0.5, 1, 5, 10, 30, 60]
67
+ end
68
+ end
69
+ end
@@ -11,19 +11,40 @@ module Rabbitek
11
11
  include Loggable
12
12
 
13
13
  def call(consumer, message)
14
- info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
14
+ log_started(consumer, message)
15
15
 
16
- start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
16
+ start = current_time
17
17
 
18
18
  super
19
19
  ensure
20
+ total_time = current_time - start
21
+
22
+ log_finished(consumer, message, total_time)
23
+ metrics_measure_time(consumer, total_time)
24
+ end
25
+
26
+ private
27
+
28
+ def current_time
29
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
30
+ end
31
+
32
+ def log_started(consumer, message)
33
+ info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
34
+ end
35
+
36
+ def log_finished(consumer, message, total_time)
20
37
  info(
21
- message: 'Finished',
38
+ message: 'Finished job',
22
39
  consumer: message.delivery_info.routing_key,
23
- time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
40
+ time: total_time,
24
41
  jid: consumer.jid
25
42
  )
26
43
  end
44
+
45
+ def metrics_measure_time(consumer, total_time)
46
+ Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
47
+ end
27
48
  end
28
49
  end
29
50
  end
@@ -0,0 +1,84 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'forwardable'
4
+
5
+ module Rabbitek
6
+ ##
7
+ # Single message processor
8
+ class MessageProcessor
9
+ include Loggable
10
+ extend Forwardable
11
+
12
+ def initialize(starter, delivery_info, properties, payload)
13
+ @starter = starter
14
+ @delivery_info = delivery_info
15
+ @properties = properties
16
+ @payload = payload
17
+ end
18
+
19
+ def process
20
+ consumer.set_context
21
+
22
+ metrics_add_processed_count
23
+
24
+ hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
25
+ hook_walker.call!(consumer, message) { |*args| run_job(*args) }
26
+ rescue StandardError => e
27
+ on_message_errored(e)
28
+ end
29
+
30
+ private
31
+
32
+ attr_reader :starter, :delivery_info, :properties, :payload
33
+ def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
34
+
35
+ def run_job(modified_consumer, message)
36
+ if modified_consumer.class.batch
37
+ run_job_batched(modified_consumer, message)
38
+ else
39
+ modified_consumer.perform(message)
40
+ modified_consumer.ack!(message.delivery_info)
41
+ end
42
+ end
43
+
44
+ def run_job_batched(modified_consumer, message)
45
+ Batcher.new(modified_consumer).perform(message) do |batch|
46
+ modified_consumer.perform(batch)
47
+ modified_consumer.ack!(batch.last.delivery_info, true)
48
+ end
49
+ end
50
+
51
+ def on_message_errored(exception)
52
+ error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
53
+ metrics_add_errored_count
54
+ end
55
+
56
+ def metrics_add_processed_count
57
+ Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
58
+ end
59
+
60
+ def metrics_add_errored_count
61
+ Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
62
+ end
63
+
64
+ def message
65
+ @message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
66
+ end
67
+
68
+ def consumer
69
+ @consumer ||= consumer_instance(message.delivery_info.routing_key)
70
+ end
71
+
72
+ def consumer_instance(routing_key)
73
+ Thread.current[:worker_classes] ||= {}
74
+ klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
75
+ klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
76
+ rescue NameError
77
+ nil # TODO: to dead queue
78
+ end
79
+
80
+ def metrics_tags
81
+ { consumer: consumer.class }
82
+ end
83
+ end
84
+ end
@@ -17,58 +17,18 @@ module Rabbitek
17
17
  setup_bindings!
18
18
 
19
19
  work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
20
- message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
21
- Rabbitek.reloader.call { on_message_received(message) }
20
+ Rabbitek.reloader.call do
21
+ MessageProcessor.new(self, delivery_info, properties, payload).process
22
+ end
22
23
  end
23
24
  end
24
25
 
25
- private
26
-
27
- attr_reader :connection, :queue_name, :consumers, :opts
28
-
29
- def setup_bindings!
30
- consumers.each do |worker_class|
31
- work_queue.bind(work_exchange, routing_key: worker_class.to_s)
32
- retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
33
- end
34
- end
35
-
36
- def on_message_received(message)
37
- consumer = consumer_instance(message.delivery_info.routing_key)
38
- consumer.set_context
39
-
40
- hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
41
-
42
- hook_walker.call!(consumer, message) do |*args|
43
- run_job(*args)
44
- end
45
- rescue StandardError => e
46
- error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
47
- end
48
-
49
- def run_job(consumer, message)
50
- if consumer.class.batch
51
- run_job_batched(consumer, message)
52
- else
53
- consumer.perform(message)
54
- consumer.ack!(message.delivery_info)
55
- end
56
- end
57
-
58
- def consumer_instance(routing_key)
59
- Thread.current[:worker_classes] ||= {}
60
- klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
61
- klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
62
- rescue NameError
63
- nil # TODO: to dead queue
64
- end
65
-
66
26
  def channel
67
27
  @channel ||= begin
68
- channel = connection.create_channel
69
- channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
70
- channel
71
- end
28
+ channel = connection.create_channel
29
+ channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
30
+ channel
31
+ end
72
32
  end
73
33
 
74
34
  def work_exchange
@@ -95,10 +55,14 @@ module Rabbitek
95
55
  )
96
56
  end
97
57
 
98
- def run_job_batched(consumer, message)
99
- Batcher.new(consumer).perform(message) do |batch|
100
- consumer.perform(batch)
101
- consumer.ack!(batch.last.delivery_info, true)
58
+ private
59
+
60
+ attr_reader :connection, :queue_name, :consumers, :opts
61
+
62
+ def setup_bindings!
63
+ consumers.each do |worker_class|
64
+ work_queue.bind(work_exchange, routing_key: worker_class.to_s)
65
+ retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
102
66
  end
103
67
  end
104
68
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Rabbitek
4
- VERSION = '0.3.5'
4
+ VERSION = '0.4.0'
5
5
  end
data/rabbitek.gemspec CHANGED
@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
34
34
  spec.add_dependency 'oj', '~> 3.6'
35
35
  spec.add_dependency 'opentracing', '~> 0.4'
36
36
  spec.add_dependency 'slop', '~> 4.0'
37
+ spec.add_dependency 'yabeda'
37
38
 
38
39
  spec.add_development_dependency 'bundler', '~> 2.0'
39
40
  spec.add_development_dependency 'pry'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rabbitek
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.5
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Boostcom
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-01-15 00:00:00.000000000 Z
11
+ date: 2020-01-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '4.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: yabeda
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  - !ruby/object:Gem::Dependency
84
98
  name: bundler
85
99
  requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
204
218
  - lib/rabbitek/server/hooks/retry.rb
205
219
  - lib/rabbitek/server/hooks/time_tracker.rb
206
220
  - lib/rabbitek/server/message.rb
221
+ - lib/rabbitek/server/message_processor.rb
207
222
  - lib/rabbitek/server/retryer.rb
208
223
  - lib/rabbitek/server/server_hook.rb
209
224
  - lib/rabbitek/server/starter.rb