sevak 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20d0243cb06ba1e8a220389877af139bc91baee3
-  data.tar.gz: 1737e507cd67efa773052cfb41e2079ae1cedda7
+  metadata.gz: 952420aaf9a1ee5df304b591f5834a91204e0d95
+  data.tar.gz: 8c49b4bf7c75b5795eb63c5352f981f9f2b1c77b
 SHA512:
-  metadata.gz: 4da67a783427a9c8efb63994a961d40a31b0f2196164c76af11e2a614570c63dd419d1588131130aaecd00cb919ce090a76b0c52e201f1adce221d6cc151a944
-  data.tar.gz: '089ff8a69948db534625f10aada9d4fe3a037acc9948d378152dd0f8da0d45dd42940c14812aec7e85860b8f5582d56dfdea3624af0edfcffcd39c41b3d813fb'
+  metadata.gz: '0982036b4ce6d2c568b8fb81192bfb496441c46aa3ff011f4365dacec48ceb66dd9d17fa8cf121e222d2bc51387a9da2d217531c9e2251f4f7197b8676d01b4e'
+  data.tar.gz: 53feaed12dc1abe46de31842be7c321e9f1f3c5da230130dbd92aeffc8a023301e9b9a8f16a115bd912f2fff66d03070fc3891d6ee8cc722b67780af118dc8b2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sevak (0.4.4)
+    sevak (0.5.0)
       bunny (~> 2.7)
 
 GEM

data/README.md

@@ -1,10 +1,13 @@
 # Current version
 
-    0.4.3
+    0.4.4
 
 Sevak gem makes it easy to send and receive messages from rabbitmq queues. It is built on top of the bunny gem.
 It also supports delayed queuing using [rabbitmq delayed exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange)
 
+
+Durable option is enabled for all the queues to have persisting queues.  
+
 # Dependencies
 
 * [rabbitmq delayed exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange):
@@ -19,6 +22,21 @@ To install this plugin:
 # Installation
 
     gem install sevak
+    
+# Make sure the rabbitmq server is running
+
+You can either install and run the rabbitmq server from the appropriate package for your os or you can run the preconfigured docker image for local testing.
+
+The image can be found here.
+
+https://hub.docker.com/r/deepakkumarnd/sevak/
+
+You can run the rabbitmq by doing the following step 
+
+    docker pull deepakkumarnd/sevak
+    docker run -d --name rabbitmq_test -p 15672:15672 -p 5672:5672 deepakkumarnd/sevak
+
+After this you will be able to access the management console by going to http://localhost:15672. You can login using guest, guest as username and password.
 
 # Configuration
 
@@ -73,3 +91,10 @@ To receive message from this queue and process the message create a consumer fil
     end
 
 The return status can have three values :ok, :error, :retry.
+
+### Run the preconfigured docker container for testing purpose
+
+https://hub.docker.com/r/deepakkumarnd/sevak/
+
+docker pull deepakkumarnd/sevak
+docker run -d --name rabbitmq_test -p 15672:15672 -p 5672:5672 deepakkumarnd/sevak

data/config/sevak.yml.example

@@ -0,0 +1,5 @@
+host: 'localhost'
+port: '5672'
+user: 'guest'
+password: 'guest'
+prefetch_count: 10

data/lib/sevak.rb

@@ -11,9 +11,28 @@ module Sevak
   class Config
 
     # allowed configurations
-    CONFIGURATION = %w(host port user password prefetch_count)
+    CONFIGURATION = %w(
+      host
+      port
+      user
+      password
+      prefetch_count
+      autoscale
+      max_process_limit
+      min_process_limit).freeze
 
     def initialize
+      @config = {
+        'host' => 'localhost',
+        'port' => '5672',
+        'user' => 'guest',
+        'password' => 'guest',
+        'prefetch_count' => 10,
+        'autoscale' => false,
+        'max_process_limit' => 10,
+        'min_process_limit' => 1
+      }
+
       load_configuration_from_yml
     end
 
@@ -36,13 +55,15 @@ module Sevak
       end
     end
 
+    def to_h
+      @config
+    end
+
     private
 
     def load_configuration_from_yml
-      @config = {}
-
       if File.exists?('config/sevak.yml')
-        @config = YAML.load(File.read('config/sevak.yml'))
+        @config = @config.merge(YAML.load(File.read('config/sevak.yml')))
       end
     end
 
@@ -64,19 +85,6 @@ module Sevak
     config
   end
 
-  def self.establish_connection
-    return @conn if @conn
-
-    @conn ||= Bunny.new(
-      host: config.host,
-      port: config.port,
-      username: config.user,
-      password: config.password)
-
-    @conn.start
-    @conn
-  end
-
   def self.get_logger
     if !Dir.exist? 'log'
       FileUtils.mkdir('log')
@@ -93,11 +101,6 @@ module Sevak
     @logger = Logger.new(logfile, 'weekly')
   end
 
-  def self.log(data)
-    @logger ||= get_logger
-    @logger.info(data.inspect)
-  end
-
   def self.testing?
     defined?(SEVAK_ENV) && (SEVAK_ENV == 'test')
   end
@@ -109,5 +112,6 @@ module Sevak
 end
 
 require 'sevak/core'
+require 'sevak/autoscale'
 require 'sevak/consumer'
 require 'sevak/publisher'

data/lib/sevak/autoscale.rb

@@ -0,0 +1,100 @@
+module Sevak
+  module Autoscale
+    Signal.trap('TERM') { Process.waitall; exit }
+    Signal.trap('INT') { Process.waitall; exit  }
+
+    def pids
+      @pids ||= []
+    end
+
+    def save_pid
+      @pids
+    end
+
+    def stop
+      exit
+    end
+
+    def fork_process
+      begin
+        pid = fork do
+          trap('HUP') { stop }
+          trap('TERM') { stop }
+          trap('INT') { stop }
+
+          self.class.new.start_worker
+        end
+
+        pids.push(pid)
+
+      rescue => e
+        log("Unable to fork process #{e.message}")
+      end
+    end
+
+    def kill_all
+      pids.each do |pid|
+        kill_process(pid)
+      end
+      pids = []
+    end
+
+    def kill_process(pid)
+      return if pid.nil?
+
+      begin
+        Process.kill("HUP", pid)
+        Process.wait
+      rescue => e
+        log("Unable to kill process #{e.message}")
+      end
+    end
+
+    def process_count
+      pids.size
+    end
+
+    def cleanup
+      kill_all
+      exit
+    end
+
+    def calculate_average_load
+      ratio = begin
+        message_count / process_count
+      rescue ZeroDivisionError => e
+        10
+      end
+
+      if ratio >= 10
+        :high
+      elsif ratio <= 2
+        :low
+      else
+        :medium
+      end
+    end
+
+    def start_master_worker
+      fork_process
+
+      loop do
+        avg_load = calculate_average_load
+
+        if (avg_load == :high) && (pids.size < config.max_process_limit)
+          fork_process
+        elsif (avg_load == :low) && (process_count > config.min_process_limit)
+          pid = pids.shift
+          kill_process(pid) if pid
+        else
+        end
+
+        sleep 5
+      end
+    rescue => e
+      cleanup
+    ensure
+      cleanup
+    end
+  end
+end

data/lib/sevak/consumer.rb

@@ -6,6 +6,7 @@ module Sevak
   class ConsumerBase
 
     include Core
+    include Autoscale
 
     DEFAULT_PREFETCH_COUNT = 10
 
@@ -36,6 +37,14 @@ module Sevak
     end
 
     def start
+      if config.autoscale
+        start_master_worker
+      else
+        start_worker
+      end
+    end
+
+    def start_worker
       channel.prefetch(config.prefetch_count || DEFAULT_PREFETCH_COUNT)
 
       queue.subscribe(manual_ack: true, exclusive: false) do |delivery_info, metadata, payload|
@@ -47,7 +56,7 @@ module Sevak
         begin
           status = run(body)
         rescue => ex
-          Sevak.log(exception_details(ex, payload))
+          log(exception_details(ex, payload))
           status = :error
         end
 
@@ -91,7 +100,7 @@ module Sevak
     def run(payload)
       # implement business logic in the corresponding consumer, the run method should respond with
       # status :ok, :error, :retry after the processing is over
-      Sevak.log("Implement run method. Payload: #{payload.inspect} #{}")
+      log("Implement run method. Payload: #{payload.inspect} #{}")
       :ok
     end
   end

data/lib/sevak/core.rb

@@ -1,7 +1,20 @@
 module Sevak
   module Core
     def connection
-      ::Sevak.establish_connection
+      return @conn if @conn
+      @conn ||= Bunny.new(
+        host: config.host,
+        port: config.port,
+        username: config.user,
+        password: config.password)
+
+      @conn.start
+      @conn
+    end
+
+    def log(data)
+        @logger ||= ::Sevak.get_logger
+        @logger.info(data.inspect)
     end
 
     def config

data/lib/sevak/version.rb

@@ -1,3 +1,3 @@
 module Sevak
-  VERSION = '0.4.4'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sevak
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Deepak Kumar
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-01 00:00:00.000000000 Z
+date: 2018-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -42,7 +42,9 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- config/sevak.yml.example
 - lib/sevak.rb
+- lib/sevak/autoscale.rb
 - lib/sevak/consumer.rb
 - lib/sevak/core.rb
 - lib/sevak/publisher.rb