rabbitek 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 48e1e5b2bc2ea1d5a8de3e5e6e35ec5f8ef3e6961a99987f544f87dfc043d48f
-  data.tar.gz: b413bb3111ab56d0cf135bc5af3c6a754c675d945cf5919932881e57f31c6d94
+SHA1:
+  metadata.gz: abffed1ec6fb3e1837dc3f75bd819742e1350935
+  data.tar.gz: 72f165622c0f6346dfacb2666db46e2a0cc787b1
 SHA512:
-  metadata.gz: 04dc3c2f72c7157d9d7eb987aa91c9e97fcbd588c2a06079e37db8fba3a5d2ef202cb52337cc9b7065d4ec711e1976a132efbd53dce5fba8f01a037b1afe9585
-  data.tar.gz: c5b4d5946c8ebf3a8f374de46ec73ce40504d32a0fed75329fc568e027b12697d19c2c2dab1562da03a9ea60abce242c9ebe907c96a8f66b5f9963fbcff0aced
+  metadata.gz: a228c331c23fe8e80d9ace1a4c7847521b79a562cc14f01f413ea419337afb08e979f1673508b247dabb2e0565571f6e2b2ae0113d6fc60e51f50fa1cd29c00f
+  data.tar.gz: ce5ad374f81f11b23345f6105261af2204ad4c19421d97c6345e35ba8f4d3ccb169697212e02ef866c057865a64aa3d2f42950ffea6f198cef54337e7569a72a

data/.rubocop.yml

@@ -3,7 +3,7 @@ AllCops:
   - 'vendor/**/*'
   - 'spec/fixtures/**/*'
   - 'tmp/**/*'
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.5
 
 Metrics/LineLength:
   Max: 120

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+## v0.2.0
+
+### Breaking changes
+
+* Retry queue now uses direct exchange with routing (prevent message duplicating on retry)
+* Change API to be much more universal
+* Auto ACK on job success
+
+### Other
+
+* Add batching
+
+## v0.1.0
+
+* Initial version

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rabbitek (0.1.0)
+    rabbitek (0.1.2)
       activesupport (> 3.0)
       bunny (~> 2.11.0)
       oj (~> 3.6)
@@ -11,7 +11,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.2.1)
+    activesupport (5.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -20,13 +20,13 @@ GEM
     ast (2.4.0)
     bunny (2.11.0)
       amq-protocol (~> 2.3.0)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.3)
     diff-lcs (1.3)
     i18n (1.1.1)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.1)
     minitest (5.11.3)
-    oj (3.7.0)
+    oj (3.7.4)
     opentracing (0.4.3)
     parallel (1.12.1)
     parser (2.5.1.2)
@@ -73,4 +73,4 @@ DEPENDENCIES
   rubocop (~> 0.58.0)
 
 BUNDLED WITH
-   1.16.4
+   1.17.1

data/README.md

@@ -50,8 +50,12 @@ and create `perform` method same way as on example:
 
     rabbit_options config_file: 'config/rabbitek.yml'
 
-    def perform(payload, delivery_info, properties)
-      ack!(delivery_info)
+    def perform(message)
+      puts "Payload: #{message.payload}"
+      puts "Delivery Info: "#{message.delivery_info}"
+      puts "Properties: "#{message.properties}"
+      
+      # Mesage will be automatically acked unless exception is raised 
     end
   end
 ```
@@ -64,13 +68,25 @@ bundle exec rabbitek
 
 You can schedule jobs e.g.: `ExampleCustomer.perform_async(some: :payload)`
 
+### Batching
+
+```
+  class ExampleConsumer
+    include Rabbitek::Consumer
+
+    rabbit_options config_file: 'config/rabbitek.yml', batch: 1000
+
+    # When batch is defined, the perform method will have batch of up to N messages yielded.
+    def perform(messages)
+    end
+  end
+```
 
 ## Roadmap
 
 * more tests!
 * dead queue
 * CRON jobs
-* job batching (run consumer with e.g. max 100 messages at once)
 * extended docs and how to
 * prometheus metrics
 

data/lib/rabbitek/cli.rb

@@ -4,13 +4,15 @@ require 'slop'
 require 'yaml'
 
 require_relative './cli/signal_handlers'
+require_relative './loggable'
+
 require 'rabbitek'
 
 module Rabbitek
   ##
   # Rabbitek server CLI
   class CLI
-    include Loggable
+    include ::Rabbitek::Loggable
 
     def run
       opts
@@ -36,7 +38,7 @@ module Rabbitek
     private
 
     def start_log # rubocop:disable Metrics/AbcSize
-      info "Rabbit consumers '[#{configuration[:workers].map(&:to_s).join(', ')}]' started with PID #{Process.pid}"
+      info "Rabbit consumers '[#{configuration[:consumers].map(&:to_s).join(', ')}]' started with PID #{Process.pid}"
       info "Client hooks: [#{Rabbitek.config.client_hooks.map(&:class).join(', ')}]"
       info "Server hooks: [#{Rabbitek.config.server_hooks.map(&:class).join(', ')}]"
     end
@@ -66,7 +68,7 @@ module Rabbitek
     end
 
     def map_consumer_workers!
-      configuration[:workers].map!(&:constantize)
+      configuration[:consumers].map!(&:constantize)
     end
 
     def boot_consumers

data/lib/rabbitek/client/hooks/opentracing.rb

@@ -12,17 +12,15 @@ module Rabbitek
           result = nil
 
           ::OpenTracing.start_active_span(params[:routing_key], opentracing_options(params)) do |scope|
-            begin
-              params[:headers] ||= {}
-              Utils::OpenTracing.inject!(scope.span, params[:headers])
+            params[:headers] ||= {}
+            Utils::OpenTracing.inject!(scope.span, params[:headers])
 
-              result = super
-            rescue StandardError => e
-              raise unless scope.span
+            result = super
+          rescue StandardError => e
+            raise unless scope.span
 
-              Utils::OpenTracing.log_error(scope.span, e)
-              raise
-            end
+            Utils::OpenTracing.log_error(scope.span, e)
+            raise
           end
 
           result

data/lib/rabbitek/config.rb

@@ -7,7 +7,8 @@ module Rabbitek
     DEFAULTS = {
       bunny_configuration: { hosts: 'localhost:5672', vhost: '/' },
       log_format: 'json',
-      enable_newrelic: true
+      enable_newrelic: true,
+      logger: Logger.new(STDOUT)
     }.freeze
 
     attr_accessor(*DEFAULTS.keys)

data/lib/rabbitek/server/batcher.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # A service to group messages from queue by batches.
+  class Batcher
+    def initialize(consumer)
+      @consumer = consumer
+      @batch_size = consumer.batch_size
+      @batch = []
+    end
+
+    def perform(message)
+      collect_batch(message)
+      yield(@batch)
+    rescue StandardError
+      retry_all_messages
+      raise
+    end
+
+    private
+
+    def collect_batch(message)
+      loop do
+        @batch << message
+        break if @batch.size >= @batch_size # stop collecting batch when maximum batch size has been reached
+
+        message = @consumer.pop_message_manually
+        break unless message # stop collecting batch when there are no more messages waiting
+      end
+    end
+
+    def retry_all_messages
+      @batch.each { |message| Rabbitek::Retryer.call(@consumer, message) }
+    end
+  end
+end

data/lib/rabbitek/server/consumer.rb

@@ -25,11 +25,11 @@ module Rabbitek
       Rabbitek.logger
     end
 
-    def parse_message(message)
-      Utils::Oj.load(message)
+    def parse_payload(payload)
+      Utils::Oj.load(payload)
     end
 
-    def perform(*_args)
+    def perform(_message)
       raise NotImplementedError
     end
 
@@ -41,11 +41,23 @@ module Rabbitek
       Thread.current[:rabbit_context][:job_id]
     end
 
+    def pop_message_manually
+      delivery_info, properties, payload = queue.pop(manual_ack: true)
+      return nil unless payload
+
+      Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+    end
+
+    def batch_size
+      self.class.batch
+    end
+
     module ClassMethods # rubocop:disable Style/Documentation
-      attr_accessor :rabbit_options_hash
+      attr_accessor :rabbit_options_hash, :batch
 
       def rabbit_options(opts)
         self.rabbit_options_hash = default_rabbit_options(opts).with_indifferent_access.merge(opts)
+        self.batch = opts[:batch]
       end
 
       def perform_async(payload, opts: {}, channel: nil)
@@ -62,7 +74,7 @@ module Rabbitek
       def perform_in(time, payload, opts: {}, channel: nil)
         publisher = Publisher.new(
           Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(rabbit_options_hash[:bind_exchange]),
-          exchange_type: :fanout,
+          exchange_type: :direct,
           channel: channel
         )
         publish_with_publisher(publisher, payload, {

data/lib/rabbitek/server/hooks/opentracing.rb

@@ -8,16 +8,16 @@ module Rabbitek
       ##
       # OpenTracing server hook
       class OpenTracing < Rabbitek::ServerHook
-        def call(consumer, delivery_info, properties, payload)
+        def call(consumer, message)
           response = nil
 
-          ::OpenTracing.start_active_span(delivery_info.routing_key, opts(delivery_info, properties)) do |scope|
-            begin
-              response = super
-            rescue StandardError => e
-              Utils::OpenTracing.log_error(scope.span, e)
-              raise
-            end
+          ::OpenTracing.start_active_span(
+            message.delivery_info.routing_key, opts(message.delivery_info, message.properties)
+          ) do |scope|
+            response = super
+          rescue StandardError => e
+            Utils::OpenTracing.log_error(scope.span, e)
+            raise
           end
 
           response

data/lib/rabbitek/server/hooks/retry.rb

@@ -8,62 +8,12 @@ module Rabbitek
       ##
       # Hook to retry failed jobs
       class Retry < Rabbitek::ServerHook
-        include Loggable
-
-        def call(consumer, delivery_info, properties, payload)
+        def call(consumer, message)
           super
         rescue StandardError
-          retry_message(consumer, payload, delivery_info, properties)
+          Retryer.call(consumer, message) unless consumer.batch_size
           raise
         end
-
-        private
-
-        def retry_message(consumer, payload, delivery_info, properties)
-          headers      = properties.headers || {}
-          dead_headers = headers.fetch('x-death', []).last || {}
-
-          retry_count  = headers.fetch('x-retry-count', 0)
-          expiration   = dead_headers.fetch('original-expiration', 1000).to_i
-
-          warn_log(retry_count, expiration, consumer)
-
-          # acknowledge existing message
-          consumer.ack!(delivery_info)
-
-          if retry_count <= 25
-            # Set the new expiration with an increasing factor
-            new_expiration = expiration * 1.5
-
-            # Publish to retry queue with new expiration
-            publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
-          else
-            publish_to_dead_queue
-          end
-        end
-
-        def warn_log(retry_count, expiration, consumer)
-          warn(
-            message: 'Failure!',
-            retry_count: retry_count,
-            expiration: expiration,
-            consumer: consumer.class.to_s,
-            jid: consumer.jid
-          )
-        end
-
-        def publish_to_retry_queue(consumer, new_expiration, delivery_info, payload, retry_count)
-          consumer.retry_or_delayed_exchange.publish(
-            payload,
-            expiration: new_expiration.to_i,
-            routing_key: delivery_info.routing_key,
-            headers: { 'x-retry-count': retry_count + 1, 'x-dead-letter-routing-key': delivery_info.routing_key }
-          )
-        end
-
-        def publish_to_dead_queue
-          # TODO: implement dead queue
-        end
       end
     end
   end

data/lib/rabbitek/server/hooks/time_tracker.rb

@@ -10,8 +10,8 @@ module Rabbitek
       class TimeTracker < Rabbitek::ServerHook
         include Loggable
 
-        def call(consumer, delivery_info, properties, payload)
-          info(message: 'Starting', consumer: delivery_info.routing_key, jid: consumer.jid)
+        def call(consumer, message)
+          info(message: 'Starting', consumer: message.delivery_info.routing_key, jid: consumer.jid)
 
           start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
@@ -19,7 +19,7 @@ module Rabbitek
         ensure
           info(
             message: 'Finished',
-            consumer: delivery_info.routing_key,
+            consumer: message.delivery_info.routing_key,
             time: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start,
             jid: consumer.jid
           )

data/lib/rabbitek/server/message.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # A model representing message that consumer receives to process
+  class Message
+    attr_reader :payload, :properties, :delivery_info, :raw_payload
+
+    # @param [Hash] payload
+    # @param [Bunny::MessageProperties] properties
+    # @param [Bunny::DeliveryInfo] delivery_info
+    def initialize(payload:, properties:, delivery_info:)
+      @payload = Utils::Oj.load(payload)
+      @properties = properties
+      @delivery_info = delivery_info
+
+      @raw_payload = payload
+    end
+  end
+end

data/lib/rabbitek/server/retryer.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Rabbitek
+  ##
+  # A service to retry a failed message consuming
+  class Retryer
+    include Loggable
+
+    def self.call(*args)
+      new(*args).call
+    end
+
+    def initialize(consumer, message)
+      @consumer = consumer
+      @message = message
+
+      headers = message.properties.headers || {}
+      dead_headers = headers.fetch('x-death', []).last || {}
+
+      @retry_count = headers.fetch('x-retry-count', 0)
+      @expiration = dead_headers.fetch('original-expiration', 1000).to_i
+    end
+
+    def call
+      warn_log
+
+      # acknowledge existing message
+      @consumer.ack!(@message.delivery_info)
+
+      if @retry_count <= 25
+        # Set the new expiration with an increasing factor
+        @expiration *= 1.5
+
+        # Publish to retry queue with new expiration
+        publish_to_retry_queue
+      else
+        publish_to_dead_queue
+      end
+    end
+
+    def warn_log
+      warn(
+        message: 'Failure!',
+        retry_count: @retry_count,
+        expiration: @expiration,
+        consumer: @consumer.class.to_s,
+        jid: @consumer.jid
+      )
+    end
+
+    def publish_to_retry_queue
+      @consumer.retry_or_delayed_exchange.publish(
+        @message.raw_payload,
+        expiration: @expiration.to_i,
+        routing_key: @message.delivery_info.routing_key,
+        headers: { 'x-retry-count': @retry_count + 1, 'x-dead-letter-routing-key': @message.delivery_info.routing_key }
+      )
+    end
+
+    def publish_to_dead_queue
+      # TODO: implement dead queue
+    end
+  end
+end

data/lib/rabbitek/server/server_hook.rb

@@ -4,8 +4,8 @@ module Rabbitek
   ##
   # Base server hook class
   class ServerHook
-    def call(consumer, delivery_info, properties, payload)
-      yield(consumer, delivery_info, properties, payload)
+    def call(consumer, message)
+      yield(consumer, message)
     end
   end
 end

data/lib/rabbitek/server/starter.rb

@@ -17,7 +17,8 @@ module Rabbitek
       setup_bindings!
 
       work_queue.subscribe(manual_ack: true) do |delivery_info, properties, payload|
-        on_message_received(delivery_info, properties, payload)
+        message = Message.new(delivery_info: delivery_info, properties: properties, payload: payload)
+        on_message_received(message)
       end
     end
 
@@ -28,27 +29,32 @@ module Rabbitek
     def setup_bindings!
       consumers.each do |worker_class|
         work_queue.bind(work_exchange, routing_key: worker_class.to_s)
+        retry_or_delayed_queue.bind(retry_or_delayed_exchange, routing_key: worker_class.to_s)
       end
-      retry_or_delayed_queue.bind(retry_or_delayed_exchange)
     end
 
-    def on_message_received(delivery_info, properties, payload)
-      consumer = consumer_instance(delivery_info.routing_key)
+    def on_message_received(message)
+      consumer = consumer_instance(message.delivery_info.routing_key)
       consumer.set_context
 
       hook_walker = Utils::HookWalker.new(Rabbitek.config.server_hooks)
 
-      hook_walker.call!(consumer, delivery_info, properties, payload) do |*args|
+      hook_walker.call!(consumer, message) do |*args|
         run_job(*args)
       end
-    end
-
-    def run_job(consumer, delivery_info, properties, payload)
-      consumer.perform(consumer.parse_message(payload), delivery_info, properties)
     rescue StandardError => e
       error(message: e.inspect, backtrace: e.backtrace, consumer: consumer.class, jid: consumer.jid)
     end
 
+    def run_job(consumer, message)
+      if consumer.class.batch
+        run_job_batched(consumer, message)
+      else
+        consumer.perform(message)
+        consumer.ack!(message.delivery_info)
+      end
+    end
+
     def consumer_instance(routing_key)
       Thread.current[:worker_classes] ||= {}
       klass = Thread.current[:worker_classes][routing_key] ||= routing_key.constantize
@@ -84,9 +90,16 @@ module Rabbitek
     def retry_or_delayed_exchange
       @retry_or_delayed_exchange ||= Utils::Common.exchange(
         channel,
-        :fanout,
+        :direct,
         Utils::RabbitObjectNames.retry_or_delayed_bind_exchange(opts[:bind_exchange])
       )
     end
+
+    def run_job_batched(consumer, message)
+      Batcher.new(consumer).perform(message) do |batch|
+        consumer.perform(batch)
+        consumer.ack!(batch.last.delivery_info, true)
+      end
+    end
   end
 end

data/lib/rabbitek/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Rabbitek
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end

data/lib/rabbitek.rb

@@ -8,6 +8,7 @@ require 'opentracing'
 require 'logger'
 
 # active_support
+require 'active_support/core_ext/module/attribute_accessors'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/core_ext/string/inflections'
 
@@ -28,7 +29,7 @@ module Rabbitek
   end
 
   def self.logger
-    @logger ||= Logger.new(STDOUT)
+    @config.logger
   end
 
   def self.create_channel

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbitek
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Boostcom
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2019-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -149,6 +149,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -167,10 +168,13 @@ files:
 - lib/rabbitek/client/publisher.rb
 - lib/rabbitek/config.rb
 - lib/rabbitek/loggable.rb
+- lib/rabbitek/server/batcher.rb
 - lib/rabbitek/server/consumer.rb
 - lib/rabbitek/server/hooks/opentracing.rb
 - lib/rabbitek/server/hooks/retry.rb
 - lib/rabbitek/server/hooks/time_tracker.rb
+- lib/rabbitek/server/message.rb
+- lib/rabbitek/server/retryer.rb
 - lib/rabbitek/server/server_hook.rb
 - lib/rabbitek/server/starter.rb
 - lib/rabbitek/utils/common.rb
@@ -199,7 +203,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: High performance background job processing