rabbitek 0.3.5 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a36ba46180d4c5e3c28d3d212ff2174c49ef00618f6d49c7defd49ec0768d5ae
-  data.tar.gz: 25167637efc19c059e73ff8796a19aef890e961ec3c7c891b2b9aba1429d1ef3
+  metadata.gz: d6e7e9681a2b0440a71d3ab9ae5f15e614034ab51c6c0339905c5130733e42ad
+  data.tar.gz: a6b4d63873a74d3026d576aa9ef6d438e67c79e8ab47cf3634572c62da93ff8d
 SHA512:
-  metadata.gz: 33d4cfcad4e769c15e556b93ba40e9c779aa16ab739a889bd2ec01728b1c84cae25096a559aaaf687875f792ad43789612903c9c6c3b4e222b5a3ba0d801df68
-  data.tar.gz: 22b95c1863b7d22d66c4b9f2eb02a18c734450ac09b3aa999dd6b5f316fa40e45f7f52b101e8924a1c2f2a1e2b8601ba6830b99c4948659825983927c393f421
+  metadata.gz: 67cf264c46c092a763e4d0b48b895305d69d915c68489a54d4bd3b220c6f4ce611147afa83d00097ac71b2870390770ebcd5b27c4338fb7550e85ee458af3bd9
+  data.tar.gz: d976bcbb478c7c90e964c4f32cf3852a529415e6a3c08414459863a9e4a434dc900734cdeddc7495824853ab4a76e0b02d97566cd2f7325b8ab0ee60bc13da59

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v0.4.0
+
+Improvements:
+* Add metrics (universal, you can use e.g. prometheus)
+
 ## v0.3.5
 
 Bugfix:

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    rabbitek (0.3.4)
+    rabbitek (0.8.0)
       activesupport (> 3.0)
       bunny (~> 2.11.0)
-      oj (~> 3.6)
+      oj (> 2.0.0)
       opentracing (~> 0.4)
       slop (~> 4.0)
+      yabeda (>= 0.6.0)
 
 GEM
   remote: https://rubygems.org/
@@ -52,16 +53,17 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    amq-protocol (2.3.0)
+    amq-protocol (2.3.2)
     arel (9.0.0)
     ast (2.4.0)
     builder (3.2.4)
     bunny (2.11.0)
       amq-protocol (~> 2.3.0)
     coderay (1.1.2)
-    concurrent-ruby (1.1.5)
+    concurrent-ruby (1.1.6)
     crass (1.0.6)
     diff-lcs (1.3)
+    dry-initializer (3.0.3)
     erubi (1.9.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -83,7 +85,7 @@ GEM
     nio4r (2.5.2)
     nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
-    oj (3.10.1)
+    oj (3.10.8)
     opentracing (0.5.0)
     parallel (1.12.1)
     parser (2.5.1.2)
@@ -143,7 +145,7 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.10.0)
-    slop (4.7.0)
+    slop (4.8.2)
     sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -159,6 +161,9 @@ GEM
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
+    yabeda (0.6.1)
+      concurrent-ruby
+      dry-initializer
 
 PLATFORMS
   ruby
@@ -173,4 +178,4 @@ DEPENDENCIES
   rubocop (~> 0.58.0)
 
 BUNDLED WITH
-   2.0.2
+   2.1.4

data/README.md

@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
 * OpenTracing (http://opentracing.io/) instrumentation
 * NewRelic instrumentation for sending errors
 * Sentry instrumentation for sending errors
+* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
 
 ## Installation
 
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
 * dead queue
 * CRON jobs
 * extended docs and how to
-* prometheus metrics
 
 
 ## Development

data/lib/rabbitek.rb

@@ -6,6 +6,7 @@ require 'bunny'
 require 'oj'
 require 'opentracing'
 require 'logger'
+require 'yabeda'
 
 # active_support
 require 'active_support/core_ext/module/attribute_accessors'
@@ -54,3 +55,15 @@ module Rabbitek
     @bunny_connection ||= BunnyConnection.initialize_connection
   end
 end
+
+Yabeda.configure do
+  group :rabbitek do
+    counter   :processed_messages_count, comment: 'Total number of all messages'
+    counter   :errored_messages_count, comment: 'Total number of errored messages'
+    histogram :processed_messages_runtime do
+      comment 'How long it takes to process message'
+      unit :seconds
+      buckets [0.1, 0.5, 1, 5, 10, 30, 60]
+    end
+  end
+end

data/lib/rabbitek/cli.rb

@@ -14,9 +14,12 @@ module Rabbitek
   class CLI
     include ::Rabbitek::Loggable
 
-    def run
+    def run # rubocop:disable Metrics/AbcSize
       opts
       require_application
+
+      Yabeda.configure! unless Yabeda.already_configured?
+
       map_consumer_workers!
 
       start_log

data/lib/rabbitek/server/batcher.rb

@@ -6,7 +6,7 @@ module Rabbitek
   class Batcher
     def initialize(consumer)
       @consumer = consumer
-      @batch_size = consumer.batch_size
+      @batch_size = consumer.opts[:batch][:of]
       @batch = []
     end
 

data/lib/rabbitek/server/consumer.rb

@@ -52,16 +52,16 @@ module Rabbitek
       Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
     end
 
-    def batch_size
-      self.class.batch
+    def opts
+      self.class.opts
     end
 
     module ClassMethods # rubocop:disable Style/Documentation
-      attr_accessor :rabbit_options_hash, :batch
+      attr_accessor :rabbit_options_hash, :opts
 
       def rabbit_options(opts)
         self.rabbit_options_hash = default_rabbit_options(opts).with_indifferent_access.merge(opts)
-        self.batch = opts[:batch]
+        self.opts = opts
       end
 
       def perform_async(payload, opts: {}, channel: nil)

data/lib/rabbitek/server/hooks/retry.rb

@@ -13,7 +13,7 @@ module Rabbitek
         def call(consumer, message)
           super
         rescue StandardError
-          retry_message(consumer, message) unless consumer.batch_size
+          retry_message(consumer, message) unless consumer.opts[:batch]
           raise
         end
 

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -11,19 +11,40 @@ module Rabbitek
         include Loggable
 
         def call(consumer, message)
-          info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+          log_started(consumer, message)
 
-          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          start = current_time
 
           super
         ensure
+          total_time = current_time - start
+
+          log_finished(consumer, message, total_time)
+          metrics_measure_time(consumer, total_time)
+        end
+
+        private
+
+        def current_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def log_started(consumer, message)
+          info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+        end
+
+        def log_finished(consumer, message, total_time)
           info(
-            message: 'Finished',
+            message: 'Finished job',
             consumer: message.delivery_info.routing_key,
-            time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
+            time: total_time,
             jid: consumer.jid
           )
         end
+
+        def metrics_measure_time(consumer, total_time)
+          Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
+        end
       end
     end
   end

data/lib/rabbitek/server/message_processor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Rabbitek
+  ##
+  # Single message processor
+  class MessageProcessor
+    include Loggable
+    extend Forwardable
+
+    def initialize(starter, delivery_info, properties, payload)
+      @starter = starter
+      @delivery_info = delivery_info
+      @properties = properties
+      @payload = payload
+    end
+
+    def process
+      consumer.set_context
+
+      metrics_add_processed_count
+
+      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
+      hook_walker.call!(consumer, message) { |*args| run_job(*args) }
+    rescue StandardError => e
+      on_message_errored(e)
+    end
+
+    private
+
+    attr_reader :starter, :delivery_info, :properties, :payload
+    def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
+
+    def run_job(modified_consumer, message)
+      if modified_consumer.opts[:batch]
+        run_job_batched(modified_consumer, message)
+      else
+        modified_consumer.perform(message)
+        modified_consumer.ack!(message.delivery_info) unless modified_consumer.opts[:manual_ack]
+      end
+    end
+
+    def run_job_batched(modified_consumer, message)
+      Batcher.new(modified_consumer).perform(message) do |batch|
+        modified_consumer.perform(batch)
+        modified_consumer.ack!(batch.last.delivery_info, true) unless modified_consumer.opts[:manual_ack]
+      end
+    end
+
+    def on_message_errored(exception)
+      error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
+      metrics_add_errored_count
+    end
+
+    def metrics_add_processed_count
+      Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def metrics_add_errored_count
+      Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def message
+      @message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+    end
+
+    def consumer
+      @consumer ||= consumer_instance(message.delivery_info.routing_key)
+    end
+
+    def consumer_instance(routing_key)
+      Thread.current[:worker_classes] ||= {}
+      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
+      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+    rescue NameError
+      nil # TODO: to dead queue
+    end
+
+    def metrics_tags
+      { consumer: consumer.class }
+    end
+  end
+end

data/lib/rabbitek/server/starter.rb

@@ -17,58 +17,18 @@ module Rabbitek
       setup_bindings!
 
       work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
-        message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
-        Rabbitek.reloader.call { on_message_received(message) }
+        Rabbitek.reloader.call do
+          MessageProcessor.new(self, delivery_info, properties, payload).process
+        end
       end
     end
 
-    private
-
-    attr_reader :connection, :queue_name, :consumers, :opts
-
-    def setup_bindings!
-      consumers.each do |worker_class|
-        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
-        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
-      end
-    end
-
-    def on_message_received(message)
-      consumer = consumer_instance(message.delivery_info.routing_key)
-      consumer.set_context
-
-      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
-
-      hook_walker.call!(consumer, message) do |*args|
-        run_job(*args)
-      end
-    rescue StandardError => e
-      error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
-    end
-
-    def run_job(consumer, message)
-      if consumer.class.batch
-        run_job_batched(consumer, message)
-      else
-        consumer.perform(message)
-        consumer.ack!(message.delivery_info)
-      end
-    end
-
-    def consumer_instance(routing_key)
-      Thread.current[:worker_classes] ||= {}
-      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
-      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
-    rescue NameError
-      nil # TODO: to dead queue
-    end
-
     def channel
       @channel ||= begin
-        channel = connection.create_channel
-        channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
-        channel
-      end
+                     channel = connection.create_channel
+                     channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
+                     channel
+                   end
     end
 
     def work_exchange
@@ -95,10 +55,14 @@ module Rabbitek
       )
     end
 
-    def run_job_batched(consumer, message)
-      Batcher.new(consumer).perform(message) do |batch|
-        consumer.perform(batch)
-        consumer.ack!(batch.last.delivery_info, true)
+    private
+
+    attr_reader :connection, :queue_name, :consumers, :opts
+
+    def setup_bindings!
+      consumers.each do |worker_class|
+        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
       end
     end
   end

data/lib/rabbitek/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rabbitek
-  VERSION = '0.3.5'
+  VERSION = '0.8.0'
 end

data/rabbitek.gemspec

@@ -31,9 +31,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport', '> 3.0'
   spec.add_dependency 'bunny', '~> 2.11.0'
-  spec.add_dependency 'oj', '~> 3.6'
+  spec.add_dependency 'oj', '> 2.0.0'
   spec.add_dependency 'opentracing', '~> 0.4'
   spec.add_dependency 'slop', '~> 4.0'
+  spec.add_dependency 'yabeda', '>= 0.6.0'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbitek
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.8.0
 platform: ruby
 authors:
 - Boostcom
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-01-15 00:00:00.000000000 Z
+date: 2020-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -42,16 +42,16 @@ dependencies:
   name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: opentracing
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
 - lib/rabbitek/server/hooks/retry.rb
 - lib/rabbitek/server/hooks/time_tracker.rb
 - lib/rabbitek/server/message.rb
+- lib/rabbitek/server/message_processor.rb
 - lib/rabbitek/server/retryer.rb
 - lib/rabbitek/server/server_hook.rb
 - lib/rabbitek/server/starter.rb
@@ -221,7 +236,7 @@ metadata:
   source_code_uri: https://github.com/Boostcom/rabbitek
   changelog_uri: https://github.com/Boostcom/rabbitek/blob/master/CHANGELOG.md
   bug_tracker_uri: https://github.com/Boostcom/rabbitek/issues
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -236,8 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: High performance background job processing
 test_files: []