rabbitek 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7908fdc1329216ae02618bee4aae76564192031c6e2e7c10016758581f665d3
-  data.tar.gz: 25265bd1ab29f07298edbd2594ac972dafdc72d9ae6585dc89460e5020ce5da0
+  metadata.gz: d1f7cc70de57fb560439fe5e93b30b76e419d8e00dcea7379c6b3fe4aa4a6325
+  data.tar.gz: 1eb6bd702c62d2e9cd44f97211cfbe6769c58db9f032073bf0361a137d5c16e9
 SHA512:
-  metadata.gz: 7674a197bbd8d963f3e39253fd779e8a9261e98a8f5145b7b0ab624a690156758a81339f2fc74e9fafd4bbcc34e934c04794c89150a221e5daed2a8c83c209b9
-  data.tar.gz: 4f04a4cbf33ec615366e891500ffebd38e4d54a9c7fc4c772c04dc602279a423541ac1c3cd045e18ee0428f490356cc41c86f05420b5eaeaff775ffd31c872bd
+  metadata.gz: 398811e481a4b197ddaea89a52448d76e158576156aacffb5c3801d0b3122bfb3df1770e15c348500fd5fe59a172b3bdb3d8f0c52e4642a9306a7d983a8c616f
+  data.tar.gz: fcc347a951e4530f723e8d754a58b111b7463cc175a3944249296db049b070898b1b44e509633c7af2c8d633c6fa2fb8c45908c3e09e6fa87c677ef327db8c5c

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## v0.4.0
+
+Improvements:
+* Add metrics (universal, you can use e.g. prometheus)
+
+## v0.3.5
+
+Bugfix:
+* Fix OpenTracing support when there is no trace_id incoming with job.
+
+## v0.3.4
+
+Improvements:
+* Logging improvement
+
+## v0.3.3
+
+Improvements:
+* Failure in retry hook will result in additional logging and nacking job with requeue: true
+
+## v0.3.2
+
 ## v0.3.1
 
 Bugfix:

data/Gemfile.lock

@@ -1,53 +1,54 @@
 PATH
   remote: .
   specs:
-    rabbitek (0.3.0)
+    rabbitek (0.4.0)
       activesupport (> 3.0)
       bunny (~> 2.11.0)
       oj (~> 3.6)
       opentracing (~> 0.4)
       slop (~> 4.0)
+      yabeda
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.2)
-      actionpack (= 5.2.2)
+    actioncable (5.2.4.1)
+      actionpack (= 5.2.4.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.2)
-      actionpack (= 5.2.2)
-      actionview (= 5.2.2)
-      activejob (= 5.2.2)
+    actionmailer (5.2.4.1)
+      actionpack (= 5.2.4.1)
+      actionview (= 5.2.4.1)
+      activejob (= 5.2.4.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.2)
-      actionview (= 5.2.2)
-      activesupport (= 5.2.2)
-      rack (~> 2.0)
+    actionpack (5.2.4.1)
+      actionview (= 5.2.4.1)
+      activesupport (= 5.2.4.1)
+      rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.2)
-      activesupport (= 5.2.2)
+    actionview (5.2.4.1)
+      activesupport (= 5.2.4.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.2)
-      activesupport (= 5.2.2)
+    activejob (5.2.4.1)
+      activesupport (= 5.2.4.1)
       globalid (>= 0.3.6)
-    activemodel (5.2.2)
-      activesupport (= 5.2.2)
-    activerecord (5.2.2)
-      activemodel (= 5.2.2)
-      activesupport (= 5.2.2)
+    activemodel (5.2.4.1)
+      activesupport (= 5.2.4.1)
+    activerecord (5.2.4.1)
+      activemodel (= 5.2.4.1)
+      activesupport (= 5.2.4.1)
       arel (>= 9.0)
-    activestorage (5.2.2)
-      actionpack (= 5.2.2)
-      activerecord (= 5.2.2)
+    activestorage (5.2.4.1)
+      actionpack (= 5.2.4.1)
+      activerecord (= 5.2.4.1)
       marcel (~> 0.3.1)
-    activesupport (5.2.2)
+    activesupport (5.2.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -55,20 +56,21 @@ GEM
     amq-protocol (2.3.0)
     arel (9.0.0)
     ast (2.4.0)
-    builder (3.2.3)
+    builder (3.2.4)
     bunny (2.11.0)
       amq-protocol (~> 2.3.0)
     coderay (1.1.2)
-    concurrent-ruby (1.1.4)
-    crass (1.0.4)
+    concurrent-ruby (1.1.5)
+    crass (1.0.6)
     diff-lcs (1.3)
-    erubi (1.8.0)
+    dry-initializer (3.0.3)
+    erubi (1.9.0)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    i18n (1.5.2)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.1)
-    loofah (2.2.3)
+    loofah (2.4.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -79,11 +81,11 @@ GEM
     mimemagic (0.3.3)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.11.3)
-    nio4r (2.4.0)
-    nokogiri (1.10.3)
+    minitest (5.14.0)
+    nio4r (2.5.2)
+    nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
-    oj (3.8.1)
+    oj (3.10.0)
     opentracing (0.5.0)
     parallel (1.12.1)
     parser (2.5.1.2)
@@ -92,30 +94,30 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rack (2.0.7)
+    rack (2.1.1)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.2)
-      actioncable (= 5.2.2)
-      actionmailer (= 5.2.2)
-      actionpack (= 5.2.2)
-      actionview (= 5.2.2)
-      activejob (= 5.2.2)
-      activemodel (= 5.2.2)
-      activerecord (= 5.2.2)
-      activestorage (= 5.2.2)
-      activesupport (= 5.2.2)
+    rails (5.2.4.1)
+      actioncable (= 5.2.4.1)
+      actionmailer (= 5.2.4.1)
+      actionpack (= 5.2.4.1)
+      actionview (= 5.2.4.1)
+      activejob (= 5.2.4.1)
+      activemodel (= 5.2.4.1)
+      activerecord (= 5.2.4.1)
+      activestorage (= 5.2.4.1)
+      activesupport (= 5.2.4.1)
       bundler (>= 1.3.0)
-      railties (= 5.2.2)
+      railties (= 5.2.4.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.0.4)
-      loofah (~> 2.2, >= 2.2.2)
-    railties (5.2.2)
-      actionpack (= 5.2.2)
-      activesupport (= 5.2.2)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (5.2.4.1)
+      actionpack (= 5.2.4.1)
+      activesupport (= 5.2.4.1)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
@@ -144,21 +146,24 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.10.0)
     slop (4.7.0)
-    sprockets (3.7.2)
+    sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    thor (0.20.3)
+    thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.5)
+    tzinfo (1.2.6)
       thread_safe (~> 0.1)
     unicode-display_width (1.4.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
+    yabeda (0.3.0)
+      concurrent-ruby
+      dry-initializer
 
 PLATFORMS
   ruby
@@ -173,4 +178,4 @@ DEPENDENCIES
   rubocop (~> 0.58.0)
 
 BUNDLED WITH
-   2.0.1
+   2.0.2

data/README.md

@@ -11,6 +11,7 @@ High performance, easy to use background job processing library for Ruby using R
 * OpenTracing (http://opentracing.io/) instrumentation
 * NewRelic instrumentation for sending errors
 * Sentry instrumentation for sending errors
+* Metrics (using [Yabeda](https://github.com/yabeda-rb/yabeda))
 
 ## Installation
 
@@ -89,7 +90,6 @@ You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
 * dead queue
 * CRON jobs
 * extended docs and how to
-* prometheus metrics
 
 
 ## Development

data/lib/rabbitek.rb

@@ -6,6 +6,7 @@ require 'bunny'
 require 'oj'
 require 'opentracing'
 require 'logger'
+require 'yabeda'
 
 # active_support
 require 'active_support/core_ext/module/attribute_accessors'
@@ -54,3 +55,15 @@ module Rabbitek
     @bunny_connection ||= BunnyConnection.initialize_connection
   end
 end
+
+Yabeda.configure do
+  group :rabbitek do
+    counter   :processed_messages_count, comment: 'Total number of all messages'
+    counter   :errored_messages_count, comment: 'Total number of errored messages'
+    histogram :processed_messages_runtime do
+      comment 'How long it takes to process message'
+      unit :seconds
+      buckets [0.1, 0.5, 1, 5, 10, 30, 60]
+    end
+  end
+end

data/lib/rabbitek/cli.rb

@@ -77,7 +77,8 @@ module Rabbitek
     end
 
     def boot_consumers
-      (1..configuration[:threads]).each_with_object([]) do |_, arr|
+      (1..configuration[:threads]).each_with_object([]) do |i, arr|
+        debug "Booting thread ##{i}"
         arr << Starter.new(Rabbitek.bunny_connection, configuration).start
       end
     end

data/lib/rabbitek/server/consumer.rb

@@ -21,6 +21,10 @@ module Rabbitek
       channel.ack(delivery_info.delivery_tag, multiple)
     end
 
+    def nack!(delivery_info, multiple = false, requeue = true)
+      channel.nack(delivery_info.delivery_tag, multiple, requeue)
+    end
+
     def logger
       Rabbitek.logger
     end

data/lib/rabbitek/server/hooks/retry.rb

@@ -8,12 +8,29 @@ module Rabbitek
       ##
       # Hook to retry failed jobs
       class Retry < Rabbitek::ServerHook
+        include Loggable
+
         def call(consumer, message)
           super
         rescue StandardError
-          Retryer.call(consumer, message) unless consumer.batch_size
+          retry_message(consumer, message) unless consumer.batch_size
           raise
         end
+
+        private
+
+        def retry_message(consumer, message)
+          Retryer.call(consumer, message)
+        rescue StandardError => e
+          error(
+            message: 'Critical error while retrying. Nacking and requeueing message',
+            consumer: message.delivery_info.routing_key,
+            jid: consumer.jid,
+            error: e.to_s
+          )
+
+          consumer.nack!(message.delivery_info)
+        end
       end
     end
   end

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -11,19 +11,40 @@ module Rabbitek
         include Loggable
 
         def call(consumer, message)
-          info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+          log_started(consumer, message)
 
-          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          start = current_time
 
           super
         ensure
+          total_time = current_time - start
+
+          log_finished(consumer, message, total_time)
+          metrics_measure_time(consumer, total_time)
+        end
+
+        private
+
+        def current_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def log_started(consumer, message)
+          info(message: 'Starting job', consumer: message.delivery_info.routing_key, jid: consumer.jid)
+        end
+
+        def log_finished(consumer, message, total_time)
           info(
-            message: 'Finished',
+            message: 'Finished job',
             consumer: message.delivery_info.routing_key,
-            time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
+            time: total_time,
             jid: consumer.jid
           )
         end
+
+        def metrics_measure_time(consumer, total_time)
+          Yabeda.rabbitek.processed_messages_runtime.measure({ consumer: consumer.class }, total_time)
+        end
       end
     end
   end

data/lib/rabbitek/server/message_processor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+
+module Rabbitek
+  ##
+  # Single message processor
+  class MessageProcessor
+    include Loggable
+    extend Forwardable
+
+    def initialize(starter, delivery_info, properties, payload)
+      @starter = starter
+      @delivery_info = delivery_info
+      @properties = properties
+      @payload = payload
+    end
+
+    def process
+      consumer.set_context
+
+      metrics_add_processed_count
+
+      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
+      hook_walker.call!(consumer, message) { |*args| run_job(*args) }
+    rescue StandardError => e
+      on_message_errored(e)
+    end
+
+    private
+
+    attr_reader :starter, :delivery_info, :properties, :payload
+    def_delegators :starter, :channel, :work_queue, :retry_or_delayed_queue, :retry_or_delayed_exchange
+
+    def run_job(modified_consumer, message)
+      if modified_consumer.class.batch
+        run_job_batched(modified_consumer, message)
+      else
+        modified_consumer.perform(message)
+        modified_consumer.ack!(message.delivery_info)
+      end
+    end
+
+    def run_job_batched(modified_consumer, message)
+      Batcher.new(modified_consumer).perform(message) do |batch|
+        modified_consumer.perform(batch)
+        modified_consumer.ack!(batch.last.delivery_info, true)
+      end
+    end
+
+    def on_message_errored(exception)
+      error(message: exception.inspect, backtrace: exception.backtrace, consumer: consumer.class, jid: consumer.jid)
+      metrics_add_errored_count
+    end
+
+    def metrics_add_processed_count
+      Yabeda.rabbitek.processed_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def metrics_add_errored_count
+      Yabeda.rabbitek.errored_messages_count.increment(metrics_tags, by: 1)
+    end
+
+    def message
+      @message ||= Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+    end
+
+    def consumer
+      @consumer ||= consumer_instance(message.delivery_info.routing_key)
+    end
+
+    def consumer_instance(routing_key)
+      Thread.current[:worker_classes] ||= {}
+      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
+      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
+    rescue NameError
+      nil # TODO: to dead queue
+    end
+
+    def metrics_tags
+      { consumer: consumer.class }
+    end
+  end
+end

data/lib/rabbitek/server/starter.rb

@@ -17,58 +17,18 @@ module Rabbitek
       setup_bindings!
 
       work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
-        message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
-        Rabbitek.reloader.call { on_message_received(message) }
+        Rabbitek.reloader.call do
+          MessageProcessor.new(self, delivery_info, properties, payload).process
+        end
       end
     end
 
-    private
-
-    attr_reader :connection, :queue_name, :consumers, :opts
-
-    def setup_bindings!
-      consumers.each do |worker_class|
-        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
-        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
-      end
-    end
-
-    def on_message_received(message)
-      consumer = consumer_instance(message.delivery_info.routing_key)
-      consumer.set_context
-
-      hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
-
-      hook_walker.call!(consumer, message) do |*args|
-        run_job(*args)
-      end
-    rescue StandardError => e
-      error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
-    end
-
-    def run_job(consumer, message)
-      if consumer.class.batch
-        run_job_batched(consumer, message)
-      else
-        consumer.perform(message)
-        consumer.ack!(message.delivery_info)
-      end
-    end
-
-    def consumer_instance(routing_key)
-      Thread.current[:worker_classes] ||= {}
-      klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
-      klass.new(channel, work_queue, retry_or_delayed_queue, retry_or_delayed_exchange)
-    rescue NameError
-      nil # TODO: to dead queue
-    end
-
     def channel
       @channel ||= begin
-        channel = connection.create_channel
-        channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
-        channel
-      end
+                     channel = connection.create_channel
+                     channel.basic_qos(opts[:basic_qos]) if opts[:basic_qos].present?
+                     channel
+                   end
     end
 
     def work_exchange
@@ -95,10 +55,14 @@ module Rabbitek
       )
     end
 
-    def run_job_batched(consumer, message)
-      Batcher.new(consumer).perform(message) do |batch|
-        consumer.perform(batch)
-        consumer.ack!(batch.last.delivery_info, true)
+    private
+
+    attr_reader :connection, :queue_name, :consumers, :opts
+
+    def setup_bindings!
+      consumers.each do |worker_class|
+        work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
       end
     end
   end

data/lib/rabbitek/utils/open_tracing.rb

@@ -55,7 +55,10 @@ module Rabbitek
         private
 
         def server_references(message_properties)
-          ::OpenTracing::Reference.follows_from(extract(message_properties))
+          ctx = extract(message_properties)
+          return unless ctx
+
+          ::OpenTracing::Reference.follows_from(ctx)
         end
 
         def extract(message_properties)

data/lib/rabbitek/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rabbitek
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
 end

data/rabbitek.gemspec

@@ -14,9 +14,15 @@ Gem::Specification.new do |spec|
   spec.description   = 'High performance background job processing'
   spec.homepage      = 'http://boostcom.no'
 
+  spec.license = 'MIT'
+
+  spec.metadata['source_code_uri'] = 'https://github.com/Boostcom/rabbitek'
+  spec.metadata['changelog_uri'] = 'https://github.com/Boostcom/rabbitek/blob/master/CHANGELOG.md'
+  spec.metadata['bug_tracker_uri'] = 'https://github.com/Boostcom/rabbitek/issues'
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
   spec.bindir        = 'exe'
@@ -28,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'oj', '~> 3.6'
   spec.add_dependency 'opentracing', '~> 0.4'
   spec.add_dependency 'slop', '~> 4.0'
+  spec.add_dependency 'yabeda'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbitek
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Boostcom
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-14 00:00:00.000000000 Z
+date: 2020-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: yabeda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +218,7 @@ files:
 - lib/rabbitek/server/hooks/retry.rb
 - lib/rabbitek/server/hooks/time_tracker.rb
 - lib/rabbitek/server/message.rb
+- lib/rabbitek/server/message_processor.rb
 - lib/rabbitek/server/retryer.rb
 - lib/rabbitek/server/server_hook.rb
 - lib/rabbitek/server/starter.rb
@@ -215,8 +230,12 @@ files:
 - lib/rabbitek/version.rb
 - rabbitek.gemspec
 homepage: http://boostcom.no
-licenses: []
-metadata: {}
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/Boostcom/rabbitek
+  changelog_uri: https://github.com/Boostcom/rabbitek/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/Boostcom/rabbitek/issues
 post_install_message: 
 rdoc_options: []
 require_paths: