rabbit_carrots 0.1.20 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e3aa34b3f6e19e3490fcba113867feb0326fd10e8597b9323d8f384a36caa55
-  data.tar.gz: 972c341f20e6bcd7d84c7cae441938f6f0f8fef2451bcb84f41f9e1d4b331e0b
+  metadata.gz: 5041ccacc59130b82ec1059d5b1f2803e3ef6f7a0ac3e09cbe8120a52c6071ee
+  data.tar.gz: 25fa5dbb545b6217a6c9a74c1ca6aaa75e86e2ba189cded3eeb1bcd93cfd1418
 SHA512:
-  metadata.gz: 6b6df93150a9535d8acff68628c548958e9f22f23cb87d183460d38672d0a135e718e840df92fd70899e2deea5e8482a17d11b5ae6b6862510a8bec3cc853855
-  data.tar.gz: 5100eab0c16d895dbc0c8e0dbf0217a49d2d37e965ab8a955c543eb657f87eab1b281d09c80908b641720b792a891cc1dd1d5db8b232144627410eea167e7bf0
+  metadata.gz: 75eb397c461cc2acf4a0d8f1ba9673f49d84dd9344681218cc2a5e800eaa979e46f24ec0ecb3fbfe6dc73d25a5c3ad72e79e4256a15fe356a2fa04366402db69
+  data.tar.gz: a2a900c8d6663e5ed2d3a022e36fd1baa4ee7fbf7c7b531bde2ae1d0efa4c023cc7d5b1b007e8bbb9c82c20edaff9dd7f1c08f499a4e03d342d1915adf26836e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rabbit_carrots (0.1.20)
+    rabbit_carrots (1.0.0)
       bunny (>= 2.22)
       connection_pool (~> 2.4)
 
@@ -80,7 +80,7 @@ GEM
       rubocop-ast (>= 1.30.0, < 2.0)
     ruby-progressbar (1.13.0)
     ruby2_keywords (0.0.5)
-    set (1.0.3)
+    set (1.1.0)
     sorted_set (1.0.3)
       rbtree
       set (~> 1.0)

data/README.md

@@ -33,6 +33,9 @@ RabbitCarrots.configure do |c|
   c.rabbitmq_password = ENV.fetch('RABBITMQ__PASSWORD', nil)
   c.rabbitmq_vhost = ENV.fetch('RABBITMQ__VHOST', nil)
   c.rabbitmq_exchange_name = ENV.fetch('RABBITMQ__EXCHANGE_NAME', nil)
+  c.automatically_recover = true
+  c.network_recovery_interval = 5
+  c.recovery_attempts = 5
   c.routing_key_mappings =  [
     { routing_keys: ['RK1', 'RK2'], queue: 'QUEUE_NAME', handler: 'CLASS HANDLER IN STRING' },
     { routing_keys: ['RK1', 'RK2'], queue: 'QUEUE_NAME', handler: 'CLASS HANDLER IN STRING' }
@@ -41,8 +44,6 @@ end
 
 ```
 
-
-
 Note that handler is a class that must implement a method named ```handle!``` that takes 4 parameters as follow: 
 
 ```ruby
@@ -53,8 +54,6 @@ class DummyEventHandler
 end
 ```
 
-
-
 Inside the handle message, you can NACK the message without re-queuing by raising ```RabbitCarrots::EventHandlers::Errors::NackMessage``` exception.
 
 To NACK and re-queue, raise ```RabbitCarrots::EventHandlers::Errors::NackAndRequeueMessage``` exception. 
@@ -65,8 +64,16 @@ Note: Any other unrescued exception raised inside ```handle!``` the that is a su
 
 ### Running
 
-Then run ```bundle exec rake rabbit_carrots:eat```.
+For better scalability and improved performance, you can run rabbit_carrots in standalone mode by invoking the following command:
+```bundle exec rake rabbit_carrots:eat```.
+
+#### Puma
+
+For small and medium sized projects, you can delegate the management of the rabbit_carrots to the Puma web server. To achieve that, add the following line to your puma.rb
+
+```plugin :rabbit_carrots```
 
+This will make sure that Puma will manage rabbit carrots as a background service and will gracefully terminate if rabbit_carrots eventually loses connection after multiple automatic recovery. 
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/puma/plugin/rabbit_carrots.rb

@@ -0,0 +1,84 @@
+# rabbit_carrots.rb
+
+require 'puma/plugin'
+
+Puma::Plugin.create do
+  attr_reader :puma_pid, :rabbit_carrots_pid, :log_writer, :core_service
+
+  def start(launcher)
+    @log_writer = launcher.log_writer
+    @puma_pid = $$
+
+    @core_service = RabbitCarrots::Core.new(logger: log_writer)
+
+    in_background do
+      monitor_rabbit_carrots
+    end
+
+    launcher.events.on_booted do
+      @rabbit_carrots_pid = fork do
+        Thread.new { monitor_puma }
+        start_rabbit_carrots_consumer
+      end
+    end
+
+    launcher.events.on_stopped { stop_rabbit_carrots }
+    launcher.events.on_restart { stop_rabbit_carrots }
+  end
+
+  private
+
+  def start_rabbit_carrots_consumer
+    core_service.start(kill_to_restart_on_standard_error: true)
+  rescue StandardError => e
+    Rails.logger.error "Error starting Rabbit Carrots: #{e.message}"
+  end
+
+  def stop_rabbit_carrots
+    return unless rabbit_carrots_pid
+
+    log 'Stopping Rabbit Carrots...'
+    core_service.request_shutdown
+    Process.kill('TERM', rabbit_carrots_pid)
+    Process.wait(rabbit_carrots_pid)
+  rescue Errno::ECHILD, Errno::ESRCH
+  end
+
+  def monitor_puma
+    monitor(:puma_dead?, 'Detected Puma has gone away, stopping Rabbit Carrots...')
+  end
+
+  def monitor_rabbit_carrots
+    monitor(:rabbit_carrots_dead?, 'Rabbits Carrot is dead, stopping Puma...')
+  end
+
+  def monitor(process_dead, message)
+    loop do
+      if send(process_dead)
+        log message
+        Process.kill('TERM', $$)
+        break
+      end
+      sleep 2
+    end
+  end
+
+  def rabbit_carrots_dead?
+    Process.waitpid(rabbit_carrots_pid, Process::WNOHANG) if rabbit_carrots_started?
+    false
+  rescue Errno::ECHILD, Errno::ESRCH
+    true
+  end
+
+  def rabbit_carrots_started?
+    rabbit_carrots_pid.present?
+  end
+
+  def puma_dead?
+    Process.ppid != puma_pid
+  end
+
+  def log(...)
+    log_writer.log(...)
+  end
+end

data/lib/rabbit_carrots/configuration.rb

@@ -9,6 +9,15 @@ module RabbitCarrots
   end
 
   class Configuration
-    attr_accessor :rabbitmq_host, :rabbitmq_port, :rabbitmq_user, :rabbitmq_password, :rabbitmq_vhost, :routing_key_mappings, :rabbitmq_exchange_name
+    attr_accessor :rabbitmq_host,
+                  :rabbitmq_port,
+                  :rabbitmq_user,
+                  :rabbitmq_password,
+                  :rabbitmq_vhost,
+                  :routing_key_mappings,
+                  :rabbitmq_exchange_name,
+                  :automatically_recover,
+                  :network_recovery_interval,
+                  :recovery_attempts
   end
 end

data/lib/rabbit_carrots/connection.rb

@@ -11,7 +11,11 @@ module RabbitCarrots
         port: RabbitCarrots.configuration.rabbitmq_port,
         user: RabbitCarrots.configuration.rabbitmq_user,
         password: RabbitCarrots.configuration.rabbitmq_password,
-        vhost: RabbitCarrots.configuration.rabbitmq_vhost
+        vhost: RabbitCarrots.configuration.rabbitmq_vhost,
+        automatically_recover: RabbitCarrots.configuration.automatically_recover || true,
+        network_recovery_interval: RabbitCarrots.configuration.network_recovery_interval || 5,
+        recovery_attempts: RabbitCarrots.configuration.recovery_attempts || 5,
+        recovery_attempts_exhausted: -> { Process.kill('TERM', Process.pid) }
       )
 
       @connection.start

data/lib/rabbit_carrots/core.rb

@@ -0,0 +1,113 @@
+module RabbitCarrots
+  class Core
+    attr_reader :logger
+
+    DatabaseAgonsticNotNullViolation = defined?(ActiveRecord) ? ActiveRecord::NotNullViolation : RabbitCarrots::EventHandlers::Errors::PlaceholderError
+    DatabaseAgonsticConnectionNotEstablished = defined?(ActiveRecord) ? ActiveRecord::ConnectionNotEstablished : Mongo::Error::SocketError
+    DatabaseAgnosticRecordInvalid = defined?(ActiveRecord) ? ActiveRecord::RecordInvalid : Mongoid::Errors::Validations
+
+    def initialize(logger: nil)
+      @logger = logger || Logger.new(Rails.env.production? ? '/proc/self/fd/1' : $stdout)
+      @threads = []
+      @running = true
+      @shutdown_requested = false
+    end
+
+    def start(kill_to_restart_on_standard_error: false)
+      channels = RabbitCarrots.configuration.routing_key_mappings.map do |mapping|
+        { **mapping, handler: mapping[:handler].constantize }
+      end
+
+      channels.each do |channel|
+        handler_class = channel[:handler]
+        raise "#{handler_class.name} must respond to `handle!`" unless handler_class.respond_to?(:handle!)
+
+        @threads << Thread.new do
+          run_task(
+            queue_name: channel[:queue],
+            handler_class:,
+            routing_keys: channel[:routing_keys],
+            queue_arguments: channel[:arguments],
+            kill_to_restart_on_standard_error:
+          )
+        end
+      end
+
+      Signal.trap('INT') { request_shutdown }
+      Signal.trap('TERM') { request_shutdown }
+
+      while @running
+        if @shutdown_requested
+          request_shutdown
+          sleep 1
+          break
+        end
+        sleep 1
+      end
+
+      @threads.each(&:join)
+    rescue StandardError => e
+      logger.error "Error starting Rabbit Carrots: #{e.message}"
+    end
+
+    def request_shutdown
+      # Workaround to a known issue with Signal Traps and logs
+      Thread.start do
+        logger.log 'Shutting down Rabbit Carrots service...'
+      end
+      @shutdown_requested = true
+      @threads.each(&:kill)
+      stop
+    end
+
+    def stop
+      # Workaround to a known issue with Signal Traps and logs
+      Thread.start do
+        logger.log 'Stoppig the Rabbit Carrots service...'
+      end
+      @running = false
+    end
+
+    def run_task(queue_name:, handler_class:, routing_keys:, queue_arguments: {}, kill_to_restart_on_standard_error: false)
+      RabbitCarrots::Connection.instance.channel.with do |channel|
+        exchange = channel.topic(RabbitCarrots.configuration.rabbitmq_exchange_name, durable: true)
+
+        logger.log "Listening on QUEUE: #{queue_name} for ROUTING KEYS: #{routing_keys}"
+        queue = channel.queue(queue_name, durable: true, arguments: queue_arguments)
+
+        routing_keys.map(&:strip).each { |k| queue.bind(exchange, routing_key: k) }
+
+        queue.subscribe(block: false, manual_ack: true, prefetch: 10) do |delivery_info, properties, payload|
+          break if @shutdown_requested
+
+          logger.log "Received from queue: #{queue_name}, Routing Keys: #{routing_keys}"
+          handler_class.handle!(channel, delivery_info, properties, payload)
+          channel.ack(delivery_info.delivery_tag, false)
+        rescue RabbitCarrots::EventHandlers::Errors::NackMessage, JSON::ParserError => _e
+          logger.log "Nacked message: #{payload}"
+          channel.nack(delivery_info.delivery_tag, false, false)
+        rescue RabbitCarrots::EventHandlers::Errors::NackAndRequeueMessage => _e
+          logger.log "Nacked and Requeued message: #{payload}"
+          channel.nack(delivery_info.delivery_tag, false, true)
+        rescue DatabaseAgonsticNotNullViolation, DatabaseAgnosticRecordInvalid => e
+          logger.log "Null constraint or Invalid violation: #{payload}. Error: #{e.message}"
+          channel.ack(delivery_info.delivery_tag, false)
+        rescue DatabaseAgonsticConnectionNotEstablished => e
+          logger.log "Error connection not established to the database: #{payload}. Error: #{e.message}"
+          sleep 3
+          channel.nack(delivery_info.delivery_tag, false, true)
+        rescue StandardError => e
+          logger.log "Error handling message: #{payload}. Error: #{e.message}"
+          sleep 3
+          channel.nack(delivery_info.delivery_tag, false, true)
+          Process.kill('SIGTERM', Process.pid) if kill_to_restart_on_standard_error
+        end
+
+        logger.log "Ending task for queue: #{queue_name}"
+      end
+    rescue StandardError => e
+      logger.error "Bunny session error: #{e.message}"
+      request_shutdown
+    end
+  end
+end

data/lib/rabbit_carrots/tasks/rmq.rake

@@ -1,77 +1,13 @@
-require 'bunny'
-
 namespace :rabbit_carrots do
-  desc 'Listener for Queue'
+  desc 'Rake task for standalone RabbitCarrots mode'
   task eat: :environment do
     Rails.application.eager_load!
 
-    # rubocop:disable Lint/ConstantDefinitionInBlock
-    DatabaseAgonsticNotNullViolation = defined?(ActiveRecord) ? ActiveRecord::NotNullViolation : RabbitCarrots::EventHandlers::Errors::PlaceholderError
-    DatabaseAgonsticConnectionNotEstablished = defined?(ActiveRecord) ? ActiveRecord::ConnectionNotEstablished : Mongo::Error::SocketError
-    DatabaseAgnosticRecordInvalid = defined?(ActiveRecord) ? ActiveRecord::RecordInvalid : Mongoid::Errors::Validations
-    # rubocop:enable Lint/ConstantDefinitionInBlock
-
-    channels = RabbitCarrots.configuration.routing_key_mappings.map do |mapping|
-      # This will be supplied in initializer. At that time, the Handler will not be available to be loaded and will throw Uninitialized Constant
-      { **mapping, handler: mapping[:handler].constantize }
-    end
-
-    Rails.logger = Logger.new(Rails.env.production? ? '/proc/self/fd/1' : $stdout)
-
-    # Run RMQ Subscriber for each channel
-    channels.each do |channel|
-      handler_class = channel[:handler]
-
-      raise "#{handler_class.name} must respond to `handle!`" unless handler_class.respond_to?(:handle!)
-
-      run_task(queue_name: channel[:queue], handler_class:, routing_keys: channel[:routing_keys], queue_arguments: channel[:arguments])
-    end
-
-    # Infinite loop to keep the process running
-    loop do
-      sleep 1
-    end
-  end
-end
-
-def run_task(queue_name:, handler_class:, routing_keys:, queue_arguments: {})
-  RabbitCarrots::Connection.instance.channel.with do |channel|
-    exchange = channel.topic(RabbitCarrots.configuration.rabbitmq_exchange_name, durable: true)
-
-    Rails.logger.info "Listening on QUEUE: #{queue_name} for ROUTING KEYS: #{routing_keys}"
-    queue = channel.queue(queue_name, durable: true, arguments: queue_arguments)
-
-    routing_keys.map(&:strip).each { |k| queue.bind(exchange, routing_key: k) }
+    logger = Logger.new(Rails.env.production? ? '/proc/self/fd/1' : $stdout)
+    logger.level = Logger::INFO
 
-    queue.subscribe(block: false, manual_ack: true, prefetch: 10) do |delivery_info, properties, payload|
-      Rails.logger.info "Received from queue: #{queue_name}, Routing Keys: #{routing_keys}"
-      handler_class.handle!(channel, delivery_info, properties, payload)
-      channel.ack(delivery_info.delivery_tag, false)
-    rescue RabbitCarrots::EventHandlers::Errors::NackMessage, JSON::ParserError => _e
-      Rails.logger.info "Nacked message: #{payload}"
-      channel.nack(delivery_info.delivery_tag, false, false)
-    rescue RabbitCarrots::EventHandlers::Errors::NackAndRequeueMessage => _e
-      Rails.logger.info "Nacked and Requeued message: #{payload}"
-      channel.nack(delivery_info.delivery_tag, false, true)
-    rescue DatabaseAgonsticNotNullViolation, DatabaseAgnosticRecordInvalid => e
-      # on null constraint violation, we want to ack the message
-      Rails.logger.error "Null constraint or Invalid violation: #{payload}. Error: #{e.message}"
-      channel.ack(delivery_info.delivery_tag, false)
-    rescue DatabaseAgonsticConnectionNotEstablished => e
-      # on connection not established, we want to requeue the message and sleep for 3 seconds
-      Rails.logger.error "Error connection not established to the database: #{payload}. Error: #{e.message}"
-      # delay for 3 seconds before requeuing
-      sleep 3
-      channel.nack(delivery_info.delivery_tag, false, true)
-    rescue StandardError => e
-      Rails.logger.error "Error handling message: #{payload}. Error: #{e.message}"
-      # requeue the message then kill the container
-      sleep 3
-      channel.nack(delivery_info.delivery_tag, false, true)
-      # kill the container with sigterm
-      Process.kill('SIGTERM', Process.pid)
-    end
+    core_service = RabbitCarrots::Core.new(logger:)
 
-    Rails.logger.info 'RUN TASK ENDED'
+    core_service.start(kill_to_restart_on_standard_error: true)
   end
 end

data/lib/rabbit_carrots/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module RabbitCarrots
-  VERSION = '0.1.20'
+  VERSION = '1.0.0'
 end

data/lib/rabbit_carrots.rb

@@ -2,6 +2,7 @@
 
 require_relative 'rabbit_carrots/version'
 require 'rabbit_carrots/connection'
+require 'rabbit_carrots/core'
 require 'rabbit_carrots/configuration'
 require 'rabbit_carrots/railtie' if defined?(Rails)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rabbit_carrots
 version: !ruby/object:Gem::Version
-  version: 0.1.20
+  version: 1.0.0
 platform: ruby
 authors:
 - Brusk Awat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-14 00:00:00.000000000 Z
+date: 2024-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -54,9 +54,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- lib/puma/plugin/rabbit_carrots.rb
 - lib/rabbit_carrots.rb
 - lib/rabbit_carrots/configuration.rb
 - lib/rabbit_carrots/connection.rb
+- lib/rabbit_carrots/core.rb
 - lib/rabbit_carrots/railtie.rb
 - lib/rabbit_carrots/tasks/rmq.rake
 - lib/rabbit_carrots/version.rb
@@ -85,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.5.9
 signing_key: 
 specification_version: 4
 summary: A simple RabbitMQ consumer task