sevak 0.4.4 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +26 -1
- data/config/sevak.yml.example +5 -0
- data/lib/sevak.rb +26 -22
- data/lib/sevak/autoscale.rb +100 -0
- data/lib/sevak/consumer.rb +11 -2
- data/lib/sevak/core.rb +14 -1
- data/lib/sevak/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 952420aaf9a1ee5df304b591f5834a91204e0d95
|
4
|
+
data.tar.gz: 8c49b4bf7c75b5795eb63c5352f981f9f2b1c77b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0982036b4ce6d2c568b8fb81192bfb496441c46aa3ff011f4365dacec48ceb66dd9d17fa8cf121e222d2bc51387a9da2d217531c9e2251f4f7197b8676d01b4e'
|
7
|
+
data.tar.gz: 53feaed12dc1abe46de31842be7c321e9f1f3c5da230130dbd92aeffc8a023301e9b9a8f16a115bd912f2fff66d03070fc3891d6ee8cc722b67780af118dc8b2
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,10 +1,13 @@
|
|
1
1
|
# Current version
|
2
2
|
|
3
|
-
0.4.
|
3
|
+
0.4.4
|
4
4
|
|
5
5
|
Sevak gem makes it easy to send and receive messages from rabbitmq queues. It is built on top of the bunny gem.
|
6
6
|
It also supports delayed queuing using [rabbitmq delayed exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange)
|
7
7
|
|
8
|
+
|
9
|
+
Durable option is enabled for all the queues to have persisting queues.
|
10
|
+
|
8
11
|
# Dependencies
|
9
12
|
|
10
13
|
* [rabbitmq delayed exchange plugin](https://github.com/rabbitmq/rabbitmq-delayed-message-exchange):
|
@@ -19,6 +22,21 @@ To install this plugin:
|
|
19
22
|
# Installation
|
20
23
|
|
21
24
|
gem install sevak
|
25
|
+
|
26
|
+
# Make sure the rabbitmq server is running
|
27
|
+
|
28
|
+
You can either install and run the rabbitmq server from the appropriate package for your os or you can run the preconfigured docker image for local testing.
|
29
|
+
|
30
|
+
The image can be found here.
|
31
|
+
|
32
|
+
https://hub.docker.com/r/deepakkumarnd/sevak/
|
33
|
+
|
34
|
+
You can run the rabbitmq by doing the following step
|
35
|
+
|
36
|
+
docker pull deepakkumarnd/sevak
|
37
|
+
docker run -d --name rabbitmq_test -p 15672:15672 -p 5672:5672 deepakkumarnd/sevak
|
38
|
+
|
39
|
+
After this you will be able to access the management console by going to http://localhost:15672. You can login using guest, guest as username and password.
|
22
40
|
|
23
41
|
# Configuration
|
24
42
|
|
@@ -73,3 +91,10 @@ To receive message from this queue and process the message create a consumer fil
|
|
73
91
|
end
|
74
92
|
|
75
93
|
The return status can have three values :ok, :error, :retry.
|
94
|
+
|
95
|
+
### Run the preconfigured docker container for testing purpose
|
96
|
+
|
97
|
+
https://hub.docker.com/r/deepakkumarnd/sevak/
|
98
|
+
|
99
|
+
docker pull deepakkumarnd/sevak
|
100
|
+
docker run -d --name rabbitmq_test -p 15672:15672 -p 5672:5672 deepakkumarnd/sevak
|
data/lib/sevak.rb
CHANGED
@@ -11,9 +11,28 @@ module Sevak
|
|
11
11
|
class Config
|
12
12
|
|
13
13
|
# allowed configurations
|
14
|
-
CONFIGURATION = %w(
|
14
|
+
CONFIGURATION = %w(
|
15
|
+
host
|
16
|
+
port
|
17
|
+
user
|
18
|
+
password
|
19
|
+
prefetch_count
|
20
|
+
autoscale
|
21
|
+
max_process_limit
|
22
|
+
min_process_limit).freeze
|
15
23
|
|
16
24
|
def initialize
|
25
|
+
@config = {
|
26
|
+
'host' => 'localhost',
|
27
|
+
'port' => '5672',
|
28
|
+
'user' => 'guest',
|
29
|
+
'password' => 'guest',
|
30
|
+
'prefetch_count' => 10,
|
31
|
+
'autoscale' => false,
|
32
|
+
'max_process_limit' => 10,
|
33
|
+
'min_process_limit' => 1
|
34
|
+
}
|
35
|
+
|
17
36
|
load_configuration_from_yml
|
18
37
|
end
|
19
38
|
|
@@ -36,13 +55,15 @@ module Sevak
|
|
36
55
|
end
|
37
56
|
end
|
38
57
|
|
58
|
+
def to_h
|
59
|
+
@config
|
60
|
+
end
|
61
|
+
|
39
62
|
private
|
40
63
|
|
41
64
|
def load_configuration_from_yml
|
42
|
-
@config = {}
|
43
|
-
|
44
65
|
if File.exists?('config/sevak.yml')
|
45
|
-
@config = YAML.load(File.read('config/sevak.yml'))
|
66
|
+
@config = @config.merge(YAML.load(File.read('config/sevak.yml')))
|
46
67
|
end
|
47
68
|
end
|
48
69
|
|
@@ -64,19 +85,6 @@ module Sevak
|
|
64
85
|
config
|
65
86
|
end
|
66
87
|
|
67
|
-
def self.establish_connection
|
68
|
-
return @conn if @conn
|
69
|
-
|
70
|
-
@conn ||= Bunny.new(
|
71
|
-
host: config.host,
|
72
|
-
port: config.port,
|
73
|
-
username: config.user,
|
74
|
-
password: config.password)
|
75
|
-
|
76
|
-
@conn.start
|
77
|
-
@conn
|
78
|
-
end
|
79
|
-
|
80
88
|
def self.get_logger
|
81
89
|
if !Dir.exist? 'log'
|
82
90
|
FileUtils.mkdir('log')
|
@@ -93,11 +101,6 @@ module Sevak
|
|
93
101
|
@logger = Logger.new(logfile, 'weekly')
|
94
102
|
end
|
95
103
|
|
96
|
-
def self.log(data)
|
97
|
-
@logger ||= get_logger
|
98
|
-
@logger.info(data.inspect)
|
99
|
-
end
|
100
|
-
|
101
104
|
def self.testing?
|
102
105
|
defined?(SEVAK_ENV) && (SEVAK_ENV == 'test')
|
103
106
|
end
|
@@ -109,5 +112,6 @@ module Sevak
|
|
109
112
|
end
|
110
113
|
|
111
114
|
require 'sevak/core'
|
115
|
+
require 'sevak/autoscale'
|
112
116
|
require 'sevak/consumer'
|
113
117
|
require 'sevak/publisher'
|
@@ -0,0 +1,100 @@
|
|
1
|
+
module Sevak
|
2
|
+
module Autoscale
|
3
|
+
Signal.trap('TERM') { Process.waitall; exit }
|
4
|
+
Signal.trap('INT') { Process.waitall; exit }
|
5
|
+
|
6
|
+
def pids
|
7
|
+
@pids ||= []
|
8
|
+
end
|
9
|
+
|
10
|
+
def save_pid
|
11
|
+
@pids
|
12
|
+
end
|
13
|
+
|
14
|
+
def stop
|
15
|
+
exit
|
16
|
+
end
|
17
|
+
|
18
|
+
def fork_process
|
19
|
+
begin
|
20
|
+
pid = fork do
|
21
|
+
trap('HUP') { stop }
|
22
|
+
trap('TERM') { stop }
|
23
|
+
trap('INT') { stop }
|
24
|
+
|
25
|
+
self.class.new.start_worker
|
26
|
+
end
|
27
|
+
|
28
|
+
pids.push(pid)
|
29
|
+
|
30
|
+
rescue => e
|
31
|
+
log("Unable to fork process #{e.message}")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def kill_all
|
36
|
+
pids.each do |pid|
|
37
|
+
kill_process(pid)
|
38
|
+
end
|
39
|
+
pids = []
|
40
|
+
end
|
41
|
+
|
42
|
+
def kill_process(pid)
|
43
|
+
return if pid.nil?
|
44
|
+
|
45
|
+
begin
|
46
|
+
Process.kill("HUP", pid)
|
47
|
+
Process.wait
|
48
|
+
rescue => e
|
49
|
+
log("Unable to kill process #{e.message}")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def process_count
|
54
|
+
pids.size
|
55
|
+
end
|
56
|
+
|
57
|
+
def cleanup
|
58
|
+
kill_all
|
59
|
+
exit
|
60
|
+
end
|
61
|
+
|
62
|
+
def calculate_average_load
|
63
|
+
ratio = begin
|
64
|
+
message_count / process_count
|
65
|
+
rescue ZeroDivisionError => e
|
66
|
+
10
|
67
|
+
end
|
68
|
+
|
69
|
+
if ratio >= 10
|
70
|
+
:high
|
71
|
+
elsif ratio <= 2
|
72
|
+
:low
|
73
|
+
else
|
74
|
+
:medium
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def start_master_worker
|
79
|
+
fork_process
|
80
|
+
|
81
|
+
loop do
|
82
|
+
avg_load = calculate_average_load
|
83
|
+
|
84
|
+
if (avg_load == :high) && (pids.size < config.max_process_limit)
|
85
|
+
fork_process
|
86
|
+
elsif (avg_load == :low) && (process_count > config.min_process_limit)
|
87
|
+
pid = pids.shift
|
88
|
+
kill_process(pid) if pid
|
89
|
+
else
|
90
|
+
end
|
91
|
+
|
92
|
+
sleep 5
|
93
|
+
end
|
94
|
+
rescue => e
|
95
|
+
cleanup
|
96
|
+
ensure
|
97
|
+
cleanup
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
data/lib/sevak/consumer.rb
CHANGED
@@ -6,6 +6,7 @@ module Sevak
|
|
6
6
|
class ConsumerBase
|
7
7
|
|
8
8
|
include Core
|
9
|
+
include Autoscale
|
9
10
|
|
10
11
|
DEFAULT_PREFETCH_COUNT = 10
|
11
12
|
|
@@ -36,6 +37,14 @@ module Sevak
|
|
36
37
|
end
|
37
38
|
|
38
39
|
def start
|
40
|
+
if config.autoscale
|
41
|
+
start_master_worker
|
42
|
+
else
|
43
|
+
start_worker
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def start_worker
|
39
48
|
channel.prefetch(config.prefetch_count || DEFAULT_PREFETCH_COUNT)
|
40
49
|
|
41
50
|
queue.subscribe(manual_ack: true, exclusive: false) do |delivery_info, metadata, payload|
|
@@ -47,7 +56,7 @@ module Sevak
|
|
47
56
|
begin
|
48
57
|
status = run(body)
|
49
58
|
rescue => ex
|
50
|
-
|
59
|
+
log(exception_details(ex, payload))
|
51
60
|
status = :error
|
52
61
|
end
|
53
62
|
|
@@ -91,7 +100,7 @@ module Sevak
|
|
91
100
|
def run(payload)
|
92
101
|
# implement business logic in the corresponding consumer, the run method should respond with
|
93
102
|
# status :ok, :error, :retry after the processing is over
|
94
|
-
|
103
|
+
log("Implement run method. Payload: #{payload.inspect} #{}")
|
95
104
|
:ok
|
96
105
|
end
|
97
106
|
end
|
data/lib/sevak/core.rb
CHANGED
@@ -1,7 +1,20 @@
|
|
1
1
|
module Sevak
|
2
2
|
module Core
|
3
3
|
def connection
|
4
|
-
|
4
|
+
return @conn if @conn
|
5
|
+
@conn ||= Bunny.new(
|
6
|
+
host: config.host,
|
7
|
+
port: config.port,
|
8
|
+
username: config.user,
|
9
|
+
password: config.password)
|
10
|
+
|
11
|
+
@conn.start
|
12
|
+
@conn
|
13
|
+
end
|
14
|
+
|
15
|
+
def log(data)
|
16
|
+
@logger ||= ::Sevak.get_logger
|
17
|
+
@logger.info(data.inspect)
|
5
18
|
end
|
6
19
|
|
7
20
|
def config
|
data/lib/sevak/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sevak
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Deepak Kumar
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-08-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bunny
|
@@ -42,7 +42,9 @@ files:
|
|
42
42
|
- Rakefile
|
43
43
|
- bin/console
|
44
44
|
- bin/setup
|
45
|
+
- config/sevak.yml.example
|
45
46
|
- lib/sevak.rb
|
47
|
+
- lib/sevak/autoscale.rb
|
46
48
|
- lib/sevak/consumer.rb
|
47
49
|
- lib/sevak/core.rb
|
48
50
|
- lib/sevak/publisher.rb
|