rabbitek 0.3.5 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a36ba46180d4c5e3c28d3d212ff2174c49ef00618f6d49c7defd49ec0768d5ae
-  data.tar.gz: 25167637efc19c059e73ff8796a19aef890e961ec3c7c891b2b9aba1429d1ef3
+  metadata.gz: d1f7cc70de57fb560439fe5e93b30b76e419d8e00dcea7379c6b3fe4aa4a6325
+  data.tar.gz: 1eb6bd702c62d2e9cd44f97211cfbe6769c58db9f032073bf0361a137d5c16e9
 SHA512:
-  metadata.gz: 33d4cfcad4e769c15e556b93ba40e9c779aa16ab739a889bd2ec01728b1c84cae25096a559aaaf687875f792ad43789612903c9c6c3b4e222b5a3ba0d801df68
-  data.tar.gz: 22b95c1863b7d22d66c4b9f2eb02a18c734450ac09b3aa999dd6b5f316fa40e45f7f52b101e8924a1c2f2a1e2b8601ba6830b99c4948659825983927c393f421
+  metadata.gz: 398811e481a4b197ddaea89a52448d76e158576156aacffb5c3801d0b3122bfb3df1770e15c348500fd5fe59a172b3bdb3d8f0c52e4642a9306a7d983a8c616f
+  data.tar.gz: fcc347a951e4530f723e8d754a58b111b7463cc175a3944249296db049b070898b1b44e509633c7af2c8d633c6fa2fb8c45908c3e09e6fa87c677ef327db8c5c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v0.4.0
+
+Improvements:
+* Add metrics (universal, you can use e.g. prometheus)
+
 ## v0.3.5
 
 Bugfix:

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    rabbitek (0.3.4)
+    rabbitek (0.4.0)
       activesupport (> 3.0)
       bunny (~> 2.11.0)
       oj (~> 3.6)
       opentracing (~> 0.4)
       slop (~> 4.0)
+      yabeda
 
 GEM
   remote: https://rubygems.org/
@@ -62,6 +63,7 @@ GEM
     concurrent-ruby (1.1.5)
     crass (1.0.6)
     diff-lcs (1.3)
+    dry-initializer (3.0.3)
     erubi (1.9.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -83,7 +85,7 @@ GEM
     nio4r (2.5.2)
     nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
-    oj (3.10.1)
+    oj (3.10.0)
     opentracing (0.5.0)
     parallel (1.12.1)
     parser (2.5.1.2)
@@ -159,6 +161,9 @@ GEM
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
+    yabeda (0.3.0)
+      concurrent-ruby
+      dry-initializer
 
 PLATFORMS
   ruby

data/README.md

@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
 * OpenTracing (http://opentracing.io/) instrumentation
 * NewRelic instrumentation for sending errors
 * Sentry instrumentation for sending errors
+* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
 
 ## Installation
 
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
 * dead queue
 * CRON jobs
 * extended docs and how to
-* prometheus metrics
 
 
 ## Development

data/lib/rabbitek.rb

@@ -6,6 +6,7 @@ require 'bunny'
 require 'oj'
 require 'opentracing'
 require 'logger'
+require 'yabeda'
 
 # active_support
 require 'active_support/core_ext/module/attribute_accessors'
@@ -54,3 +55,15 @@ module Rabbitek
     @bunny_connection ||= BunnyConnection.initialize_connection
   end
 end
+
+Yabeda.configure do
+  group :rabbitek do
+    counter   :processed_messages_count, comment: 'Total number of all messages'
+    counter   :errored_messages_count, comment: 'Total number of errored messages'
+    histogram :processed_messages_runtime do
+      comment 'How long it takes to process message'
+      unit :seconds
+      buckets [0.1, 0.5, 1, 5, 10, 30, 60]
+    end
+  end
+end

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -11,19 +11,40 @@ module Rabbitek
         include Loggable
 
         def call(consumer, message)
-          info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+          log_started(consumer, message)
 
-          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          start = current_time
 
           super
         ensure
+          total_time = current_time - start
+
+          log_finished(consumer, message, total_time)
+          metrics_measure_time(consumer, total_time)
+        end
+
+        private
+
+        def current_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def log_started(consumer, message)
+          info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+        end
+
+        def log_finished(consumer, message, total_time)
           info(
-            message: 'Finished',
+            message: 'Finished job',
             consumer: message.delivery_info.routing_key,
-            time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
+            time: total_time,
             jid: consumer.jid
           )
         end
+
+        def metrics_measure_time(consumer, total_time)
+          Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
+        end
       end
     end
   end

data/lib/rabbitek/server/message_processor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Rabbitek
+  ##
+  # Single message processor
+  class MessageProcessor
+    include Loggable
+    extend Forwardable
+
+    def initialize(starter, delivery_info, properties, payload)
+      @starter = starter
+      @delivery_info = delivery_info
+      @properties = properties
+      @payload = payload
+    end
+
+    def process
+      consumer.set_context
+
+      metrics_add_processed_count
+
+      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
+      hook_walker.call!(consumer, message) { |*args| run_job(*args) }
+    rescue StandardError => e
+      on_message_errored(e)
+    end
+
+    private
+
+    attr_reader :starter, :delivery_info, :properties, :payload
+    def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
+
+    def run_job(modified_consumer, message)
+      if modified_consumer.class.batch
+        run_job_batched(modified_consumer, message)
+      else
+        modified_consumer.perform(message)
+        modified_consumer.ack!(message.delivery_info)
+      end
+    end
+
+    def run_job_batched(modified_consumer, message)
+      Batcher.new(modified_consumer).perform(message) do |batch|
+        modified_consumer.perform(batch)
+        modified_consumer.ack!(batch.last.delivery_info, true)
+      end
+    end
+
+    def on_message_errored(exception)
+      error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
+      metrics_add_errored_count
+    end
+
+    def metrics_add_processed_count
+      Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def metrics_add_errored_count
+      Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def message
+      @message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+    end
+
+    def consumer
+      @consumer ||= consumer_instance(message.delivery_info.routing_key)
+    end
+
+    def consumer_instance(routing_key)
+      Thread.current[:worker_classes] ||= {}
+      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
+      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+    rescue NameError
+      nil # TODO: to dead queue
+    end
+
+    def metrics_tags
+      { consumer: consumer.class }
+    end
+  end
+end

data/lib/rabbitek/server/starter.rb

@@ -17,58 +17,18 @@ module Rabbitek
       setup_bindings!
 
       work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
-        message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
-        Rabbitek.reloader.call { on_message_received(message) }
+        Rabbitek.reloader.call do
+          MessageProcessor.new(self, delivery_info, properties, payload).process
+        end
       end
     end
 
-    private
-
-    attr_reader :connection, :queue_name, :consumers, :opts
-
-    def setup_bindings!
-      consumers.each do |worker_class|
-        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
-        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
-      end
-    end
-
-    def on_message_received(message)
-      consumer = consumer_instance(message.delivery_info.routing_key)
-      consumer.set_context
-
-      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
-
-      hook_walker.call!(consumer, message) do |*args|
-        run_job(*args)
-      end
-    rescue StandardError => e
-      error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
-    end
-
-    def run_job(consumer, message)
-      if consumer.class.batch
-        run_job_batched(consumer, message)
-      else
-        consumer.perform(message)
-        consumer.ack!(message.delivery_info)
-      end
-    end
-
-    def consumer_instance(routing_key)
-      Thread.current[:worker_classes] ||= {}
-      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
-      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
-    rescue NameError
-      nil # TODO: to dead queue
-    end
-
     def channel
       @channel ||= begin
-        channel = connection.create_channel
-        channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
-        channel
-      end
+                     channel = connection.create_channel
+                     channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
+                     channel
+                   end
     end
 
     def work_exchange
@@ -95,10 +55,14 @@ module Rabbitek
       )
     end
 
-    def run_job_batched(consumer, message)
-      Batcher.new(consumer).perform(message) do |batch|
-        consumer.perform(batch)
-        consumer.ack!(batch.last.delivery_info, true)
+    private
+
+    attr_reader :connection, :queue_name, :consumers, :opts
+
+    def setup_bindings!
+      consumers.each do |worker_class|
+        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
       end
     end
   end

data/lib/rabbitek/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rabbitek
-  VERSION = '0.3.5'
+  VERSION = '0.4.0'
 end

data/rabbitek.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'oj', '~> 3.6'
   spec.add_dependency 'opentracing', '~> 0.4'
   spec.add_dependency 'slop', '~> 4.0'
+  spec.add_dependency 'yabeda'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbitek
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.4.0
 platform: ruby
 authors:
 - Boostcom
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-15 00:00:00.000000000 Z
+date: 2020-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
 - lib/rabbitek/server/hooks/retry.rb
 - lib/rabbitek/server/hooks/time_tracker.rb
 - lib/rabbitek/server/message.rb
+- lib/rabbitek/server/message_processor.rb
 - lib/rabbitek/server/retryer.rb
 - lib/rabbitek/server/server_hook.rb
 - lib/rabbitek/server/starter.rb