rabbitek 0.3.2 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78ba89a7e1863c0c5db9b8abb89b59d145075c3250fbf4bb679e854e8f62a181
-  data.tar.gz: c3ecf9feae5fd64141d354a622db4946286297113db75c3fbfa5eef242bd3116
+  metadata.gz: 19938345d3105a6f1fc72afaab8a04c1df84d6a528b8af255c9173facc119329
+  data.tar.gz: 579f14f003bc200d073d0a2ceec1f6c44483939e3ec82905cfb98f4cd49c7c16
 SHA512:
-  metadata.gz: '0311083dcdd6d29b9f6635fd3d8c11ab09b0a6237c76400a12c4856794412069901be0e80f058de46b094b50b0fccd3c77f67d570d91a27c632fca07335788df'
-  data.tar.gz: 411bdd19f9822adbca577ce86d13e47fbc7b0f0b615de0cd8aa0a42a46b2d6aeb7e88ba7955614749aa8d686975b5bf034d83103760455e6a5c35744a905561c
+  metadata.gz: 18a5fd3d4bf1f947b0fe8935c48a7dfa37d831fa4eea9396caa935ebb63ec9b04c05d518a28ebf68f11e3b34f76cb4b55a0c58886b465d12ab2c911ce5f9b45b
+  data.tar.gz: 6e8ffe529da3cb2a1fdccd75f5d2de6ef0b3bee54de8b38854f09e659f5c91a160bfce09f89240b7d46e3c997e6c35446c9f5129481e755af08c5c7126273801

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## v0.4.0
+
+Improvements:
+* Add metrics (universal, you can use e.g. prometheus)
+
+## v0.3.5
+
+Bugfix:
+* Fix OpenTracing support when there is no trace_id incoming with job.
+
+## v0.3.4
+
+Improvements:
+* Logging improvement
+
+## v0.3.3
+
+Improvements:
+* Failure in retry hook will result in additional logging and nacking job with requeue: true
+
 ## v0.3.2
 
 ## v0.3.1

data/Gemfile.lock

@@ -1,12 +1,13 @@
 PATH
   remote: .
   specs:
-    rabbitek (0.3.2)
+    rabbitek (0.4.0)
       activesupport (> 3.0)
       bunny (~> 2.11.0)
       oj (~> 3.6)
       opentracing (~> 0.4)
       slop (~> 4.0)
+      yabeda
 
 GEM
   remote: https://rubygems.org/
@@ -62,6 +63,7 @@ GEM
     concurrent-ruby (1.1.5)
     crass (1.0.6)
     diff-lcs (1.3)
+    dry-initializer (3.0.3)
     erubi (1.9.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
@@ -159,6 +161,9 @@ GEM
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
+    yabeda (0.3.0)
+      concurrent-ruby
+      dry-initializer
 
 PLATFORMS
   ruby

data/README.md

@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
 * OpenTracing (http://opentracing.io/) instrumentation
 * NewRelic instrumentation for sending errors
 * Sentry instrumentation for sending errors
+* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
 
 ## Installation
 
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
 * dead queue
 * CRON jobs
 * extended docs and how to
-* prometheus metrics
 
 
 ## Development

data/lib/rabbitek.rb

@@ -6,6 +6,7 @@ require 'bunny'
 require 'oj'
 require 'opentracing'
 require 'logger'
+require 'yabeda'
 
 # active_support
 require 'active_support/core_ext/module/attribute_accessors'
@@ -54,3 +55,15 @@ module Rabbitek
     @bunny_connection ||= BunnyConnection.initialize_connection
   end
 end
+
+Yabeda.configure do
+  group :rabbitek do
+    counter   :processed_messages_count, comment: 'Total number of all messages'
+    counter   :errored_messages_count, comment: 'Total number of errored messages'
+    histogram :processed_messages_runtime do
+      comment 'How long it takes to process message'
+      unit :seconds
+      buckets [0.1, 0.5, 1, 5, 10, 30, 60]
+    end
+  end
+end

data/lib/rabbitek/cli.rb

@@ -77,7 +77,8 @@ module Rabbitek
     end
 
     def boot_consumers
-      (1..configuration[:threads]).each_with_object([]) do |_, arr|
+      (1..configuration[:threads]).each_with_object([]) do |i, arr|
+        debug "Booting thread ##{i}"
         arr << Starter.new(Rabbitek.bunny_connection, configuration).start
       end
     end

data/lib/rabbitek/server/consumer.rb

@@ -21,6 +21,10 @@ module Rabbitek
       channel.ack(delivery_info.delivery_tag, multiple)
     end
 
+    def nack!(delivery_info, multiple = false, requeue = true)
+      channel.nack(delivery_info.delivery_tag, multiple, requeue)
+    end
+
     def logger
       Rabbitek.logger
     end

data/lib/rabbitek/server/hooks/retry.rb

@@ -8,12 +8,29 @@ module Rabbitek
       ##
       # Hook to retry failed jobs
       class Retry < Rabbitek::ServerHook
+        include Loggable
+
         def call(consumer, message)
           super
         rescue StandardError
-          Retryer.call(consumer, message) unless consumer.batch_size
+          retry_message(consumer, message) unless consumer.batch_size
           raise
         end
+
+        private
+
+        def retry_message(consumer, message)
+          Retryer.call(consumer, message)
+        rescue StandardError => e
+          error(
+            message: 'Critical error while retrying. Nacking and requeueing message',
+            consumer: message.delivery_info.routing_key,
+            jid: consumer.jid,
+            error: e.to_s
+          )
+
+          consumer.nack!(message.delivery_info)
+        end
       end
     end
   end

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -11,19 +11,40 @@ module Rabbitek
         include Loggable
 
         def call(consumer, message)
-          info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+          log_started(consumer, message)
 
-          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          start = current_time
 
           super
         ensure
+          total_time = current_time - start
+
+          log_finished(consumer, message, total_time)
+          metrics_measure_time(consumer, total_time)
+        end
+
+        private
+
+        def current_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def log_started(consumer, message)
+          info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+        end
+
+        def log_finished(consumer, message, total_time)
           info(
-            message: 'Finished',
+            message: 'Finished job',
             consumer: message.delivery_info.routing_key,
-            time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
+            time: total_time,
             jid: consumer.jid
           )
         end
+
+        def metrics_measure_time(consumer, total_time)
+          Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
+        end
       end
     end
   end

data/lib/rabbitek/server/message_processor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Rabbitek
+  ##
+  # Single message processor
+  class MessageProcessor
+    include Loggable
+    extend Forwardable
+
+    def initialize(starter, delivery_info, properties, payload)
+      @starter = starter
+      @delivery_info = delivery_info
+      @properties = properties
+      @payload = payload
+    end
+
+    def process
+      consumer.set_context
+
+      metrics_add_processed_count
+
+      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
+      hook_walker.call!(consumer, message) { |*args| run_job(*args) }
+    rescue StandardError => e
+      on_message_errored(e)
+    end
+
+    private
+
+    attr_reader :starter, :delivery_info, :properties, :payload
+    def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
+
+    def run_job(modified_consumer, message)
+      if modified_consumer.class.batch
+        run_job_batched(modified_consumer, message)
+      else
+        modified_consumer.perform(message)
+        modified_consumer.ack!(message.delivery_info)
+      end
+    end
+
+    def run_job_batched(modified_consumer, message)
+      Batcher.new(modified_consumer).perform(message) do |batch|
+        modified_consumer.perform(batch)
+        modified_consumer.ack!(batch.last.delivery_info, true)
+      end
+    end
+
+    def on_message_errored(exception)
+      error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
+      metrics_add_errored_count
+    end
+
+    def metrics_add_processed_count
+      Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def metrics_add_errored_count
+      Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def message
+      @message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+    end
+
+    def consumer
+      @consumer ||= consumer_instance(message.delivery_info.routing_key)
+    end
+
+    def consumer_instance(routing_key)
+      Thread.current[:worker_classes] ||= {}
+      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
+      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+    rescue NameError
+      nil # TODO: to dead queue
+    end
+
+    def metrics_tags
+      { consumer: consumer.class }
+    end
+  end
+end

data/lib/rabbitek/server/starter.rb

@@ -17,58 +17,18 @@ module Rabbitek
       setup_bindings!
 
       work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
-        message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
-        Rabbitek.reloader.call { on_message_received(message) }
+        Rabbitek.reloader.call do
+          MessageProcessor.new(self, delivery_info, properties, payload).process
+        end
       end
     end
 
-    private
-
-    attr_reader :connection, :queue_name, :consumers, :opts
-
-    def setup_bindings!
-      consumers.each do |worker_class|
-        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
-        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
-      end
-    end
-
-    def on_message_received(message)
-      consumer = consumer_instance(message.delivery_info.routing_key)
-      consumer.set_context
-
-      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
-
-      hook_walker.call!(consumer, message) do |*args|
-        run_job(*args)
-      end
-    rescue StandardError => e
-      error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
-    end
-
-    def run_job(consumer, message)
-      if consumer.class.batch
-        run_job_batched(consumer, message)
-      else
-        consumer.perform(message)
-        consumer.ack!(message.delivery_info)
-      end
-    end
-
-    def consumer_instance(routing_key)
-      Thread.current[:worker_classes] ||= {}
-      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
-      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
-    rescue NameError
-      nil # TODO: to dead queue
-    end
-
     def channel
       @channel ||= begin
-        channel = connection.create_channel
-        channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
-        channel
-      end
+                     channel = connection.create_channel
+                     channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
+                     channel
+                   end
     end
 
     def work_exchange
@@ -95,10 +55,14 @@ module Rabbitek
       )
     end
 
-    def run_job_batched(consumer, message)
-      Batcher.new(consumer).perform(message) do |batch|
-        consumer.perform(batch)
-        consumer.ack!(batch.last.delivery_info, true)
+    private
+
+    attr_reader :connection, :queue_name, :consumers, :opts
+
+    def setup_bindings!
+      consumers.each do |worker_class|
+        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
       end
     end
   end

data/lib/rabbitek/utils/open_tracing.rb

@@ -55,7 +55,10 @@ module Rabbitek
         private
 
         def server_references(message_properties)
-          ::OpenTracing::Reference.follows_from(extract(message_properties))
+          ctx = extract(message_properties)
+          return unless ctx
+
+          ::OpenTracing::Reference.follows_from(ctx)
         end
 
         def extract(message_properties)

data/lib/rabbitek/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rabbitek
-  VERSION = '0.3.2'
+  VERSION = '0.4.1'
 end

data/rabbitek.gemspec

@@ -31,9 +31,10 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport', '> 3.0'
   spec.add_dependency 'bunny', '~> 2.11.0'
-  spec.add_dependency 'oj', '~> 3.6'
+  spec.add_dependency 'oj', '> 2.0.0'
   spec.add_dependency 'opentracing', '~> 0.4'
   spec.add_dependency 'slop', '~> 4.0'
+  spec.add_dependency 'yabeda'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbitek
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.1
 platform: ruby
 authors:
 - Boostcom
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-14 00:00:00.000000000 Z
+date: 2020-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -42,16 +42,16 @@ dependencies:
   name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: opentracing
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
 - lib/rabbitek/server/hooks/retry.rb
 - lib/rabbitek/server/hooks/time_tracker.rb
 - lib/rabbitek/server/message.rb
+- lib/rabbitek/server/message_processor.rb
 - lib/rabbitek/server/retryer.rb
 - lib/rabbitek/server/server_hook.rb
 - lib/rabbitek/server/starter.rb
@@ -236,7 +251,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubyforge_project: 
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: High performance background job processing