rabbit-wq 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +21 -0
- data/bin/rabbit-wq +1 -0
- data/lib/ansi.rb +36 -0
- data/lib/rabbit_wq.rb +13 -10
- data/lib/rabbit_wq/cli.rb +98 -76
- data/lib/rabbit_wq/configuration.rb +60 -6
- data/lib/rabbit_wq/logging.rb +8 -0
- data/lib/rabbit_wq/message_handler.rb +12 -7
- data/lib/rabbit_wq/queues.rb +5 -3
- data/lib/rabbit_wq/server.rb +49 -8
- data/lib/rabbit_wq/server_daemon.rb +144 -0
- data/lib/rabbit_wq/server_logging.rb +16 -19
- data/lib/rabbit_wq/version.rb +1 -1
- data/lib/rabbit_wq/work.rb +63 -20
- data/lib/rabbit_wq/work_logger.rb +47 -0
- data/lib/rabbit_wq/worker.rb +21 -0
- data/rabbit-wq.gemspec +1 -0
- metadata +22 -9
- data/Gemfile +0 -4
data/README.md
CHANGED
@@ -69,3 +69,24 @@ Auto-scale will set up retries at the following intervals: 1 min, 5 mins, 15 min
|
|
69
69
|
|
70
70
|
Once a worker has thrown an exception and no retry attempts are remaining, the worker is placed on
|
71
71
|
the error queue with the exception type, message and backtraces.
|
72
|
+
|
73
|
+
### Logging
|
74
|
+
|
75
|
+
RabbitWQ provides a work logger that is available within all workers. You must send a reference to self
|
76
|
+
so that the #object_id may be put into the log message.
|
77
|
+
|
78
|
+
RabbitWQ.work_logger.info( self, 'Some message' )
|
79
|
+
|
80
|
+
When the RabbitWQ::Worker module is mixed into a worker you can use the logging convenience methods. You
|
81
|
+
do not have to provide a refrence to self in this case.
|
82
|
+
|
83
|
+
class SomeWorker < Struct.new( :some_variable )
|
84
|
+
include RabbitWQ::Worker
|
85
|
+
|
86
|
+
def call
|
87
|
+
info( 'Some message' )
|
88
|
+
# do some work
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
The RabbitWQ loggers provide the following log levels: debug, info, warn, error and fatal.
|
data/bin/rabbit-wq
CHANGED
data/lib/ansi.rb
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
class ANSI
|
2
|
+
|
3
|
+
def self.resolve_text( color, &block )
|
4
|
+
text = nil
|
5
|
+
if block_given?
|
6
|
+
text = block.call + reset
|
7
|
+
end
|
8
|
+
"\e[#{chart[color.to_sym]}m#{text}"
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.reset
|
12
|
+
"\e[0m"
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.chart
|
16
|
+
{
|
17
|
+
black: 30,
|
18
|
+
red: 31,
|
19
|
+
green: 32,
|
20
|
+
yellow: 33,
|
21
|
+
blue: 34,
|
22
|
+
magenta: 35,
|
23
|
+
cyan: 36,
|
24
|
+
white: 37
|
25
|
+
}
|
26
|
+
end
|
27
|
+
|
28
|
+
chart.keys.each do |color|
|
29
|
+
|
30
|
+
define_singleton_method color do |&block|
|
31
|
+
resolve_text color, &block
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
data/lib/rabbit_wq.rb
CHANGED
@@ -1,16 +1,12 @@
|
|
1
|
-
require
|
1
|
+
require 'ansi'
|
2
|
+
require 'rabbit_wq/version'
|
2
3
|
|
3
4
|
module RabbitWQ
|
4
5
|
|
5
|
-
APP_ID =
|
6
|
-
APP_NAME =
|
7
|
-
|
8
|
-
DELAY_EXCHANGE_PREFIX = "work-delay" # TODO: Make this configurable (from ENV, or file?)
|
9
|
-
ERROR_QUEUE = "work-error"
|
10
|
-
INT = "INT"
|
11
|
-
QUEUE = "work" # TODO: Make this configurable (from ENV, or file?)
|
6
|
+
APP_ID = 'rabbit-wq'
|
7
|
+
APP_NAME = 'Rabbit Work Queue'
|
8
|
+
INT = 'INT'
|
12
9
|
VERSION_COPYRIGHT = "v#{VERSION} \u00A9#{Time.now.year}"
|
13
|
-
WORK_EXCHANGE = "work" # TODO: Make this configurable (from ENV, or file?)
|
14
10
|
|
15
11
|
autoload :Command, 'rabbit_wq/command'
|
16
12
|
autoload :Configuration, 'rabbit_wq/configuration'
|
@@ -18,20 +14,27 @@ module RabbitWQ
|
|
18
14
|
autoload :Queues, 'rabbit_wq/queues'
|
19
15
|
autoload :MessageHandler, 'rabbit_wq/message_handler'
|
20
16
|
autoload :Server, 'rabbit_wq/server'
|
17
|
+
autoload :ServerDaemon, 'rabbit_wq/server_daemon'
|
21
18
|
autoload :ServerLogging, 'rabbit_wq/server_logging'
|
22
19
|
autoload :Work, 'rabbit_wq/work'
|
23
20
|
autoload :Worker, 'rabbit_wq/worker'
|
21
|
+
autoload :WorkLogger, 'rabbit_wq/work_logger'
|
24
22
|
|
25
23
|
def self.configuration
|
26
24
|
@configuration ||= Configuration.new
|
27
25
|
end
|
28
26
|
|
27
|
+
def self.configuration=( configuration )
|
28
|
+
@configuration = configuration
|
29
|
+
end
|
30
|
+
|
29
31
|
def self.configure
|
30
32
|
yield( configuration ) if block_given?
|
31
33
|
end
|
32
34
|
|
33
35
|
class << self
|
34
|
-
attr_accessor :logger
|
36
|
+
attr_accessor :logger,
|
37
|
+
:work_logger
|
35
38
|
end
|
36
39
|
|
37
40
|
end
|
data/lib/rabbit_wq/cli.rb
CHANGED
@@ -1,10 +1,13 @@
|
|
1
|
-
require 'rubygems'
|
1
|
+
require 'rubygems'
|
2
2
|
require 'rabbit_wq'
|
3
3
|
require 'trollop'
|
4
4
|
require 'yell'
|
5
5
|
|
6
6
|
module RabbitWQ
|
7
|
-
|
7
|
+
class Cli
|
8
|
+
|
9
|
+
attr_reader :cmd,
|
10
|
+
:options
|
8
11
|
|
9
12
|
SUB_COMMANDS = %w(
|
10
13
|
restart
|
@@ -13,95 +16,114 @@ module RabbitWQ
|
|
13
16
|
stop
|
14
17
|
)
|
15
18
|
|
16
|
-
|
19
|
+
DEFAULT_CONFIG_PATH = "/etc/#{APP_ID}/#{APP_ID}.conf"
|
20
|
+
DEFAULT_LOG_PATH = "/var/log/#{APP_ID}/#{APP_ID}.log"
|
21
|
+
DEFAULT_PID_PATH = "/var/run/#{APP_ID}/#{APP_ID}.pid"
|
22
|
+
|
23
|
+
DEFAULT_NUMBER_OF_THREADS = 1
|
24
|
+
|
25
|
+
def initialize( args )
|
26
|
+
Trollop::options do
|
27
|
+
version VERSION_COPYRIGHT
|
28
|
+
banner <<-EOS
|
29
|
+
#{APP_NAME} #{VERSION_COPYRIGHT}
|
30
|
+
|
31
|
+
Usage:
|
32
|
+
#{APP_ID} [command] [options]
|
33
|
+
|
34
|
+
commands:
|
35
|
+
#{SUB_COMMANDS.map { |sub_cmd| " #{sub_cmd}" }.join( "\n" )}
|
36
|
+
|
37
|
+
(For help with a command: #{APP_ID} [command] -h)
|
38
|
+
|
39
|
+
options:
|
40
|
+
EOS
|
41
|
+
stop_on SUB_COMMANDS
|
42
|
+
end
|
43
|
+
|
44
|
+
# Get the sub-command and its options
|
45
|
+
#
|
46
|
+
@cmd = ARGV.shift || ''
|
47
|
+
@options = case( cmd )
|
48
|
+
when "restart"
|
49
|
+
Trollop::options do
|
50
|
+
opt :config, "The path for the config file", :type => String, :short => '-c', :default => DEFAULT_CONFIG_PATH
|
51
|
+
opt :log_level, "The log level", :type => String, :default => 'info'
|
52
|
+
opt :log, "The path for the log file", :type => String, :short => '-l', :default => DEFAULT_LOG_PATH
|
53
|
+
opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
54
|
+
opt :threads, "The number of threads", :type => Integer, :default => DEFAULT_NUMBER_OF_THREADS, :short => '-t'
|
55
|
+
end
|
56
|
+
when "start"
|
57
|
+
Trollop::options do
|
58
|
+
opt :config, "The path for the config file", :type => String, :short => '-c', :default => DEFAULT_CONFIG_PATH
|
59
|
+
opt :interactive, "Execute the server in interactive mode", :short => '-i'
|
60
|
+
opt :log_level, "The log level", :type => String, :default => 'info'
|
61
|
+
opt :log, "The path for the log file", :type => String, :short => '-l', :default => DEFAULT_LOG_PATH
|
62
|
+
opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
63
|
+
opt :threads, "The number of threads", :type => Integer, :default => DEFAULT_NUMBER_OF_THREADS, :short => '-t'
|
64
|
+
end
|
65
|
+
when "status"
|
66
|
+
Trollop::options do
|
67
|
+
opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
68
|
+
end
|
69
|
+
when "stop"
|
70
|
+
Trollop::options do
|
71
|
+
opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
72
|
+
end
|
73
|
+
else
|
74
|
+
Trollop::die "unknown command #{cmd.inspect}"
|
75
|
+
end
|
76
|
+
|
77
|
+
if cmd == 'start'
|
78
|
+
unless options[:interactive]
|
79
|
+
Trollop::die( :config, "is required when running as daemon" ) unless options[:config]
|
80
|
+
Trollop::die( :log, "is required when running as daemon" ) unless options[:log]
|
81
|
+
Trollop::die( :pid, "is required when running as daemon" ) unless options[:pid]
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
if %w(restart status stop).include?( cmd )
|
86
|
+
Trollop::die( :pid, "is required" ) unless options[:pid]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def run
|
91
|
+
send( cmd )
|
92
|
+
end
|
93
|
+
|
94
|
+
protected
|
95
|
+
|
96
|
+
def start
|
17
97
|
if options[:interactive]
|
18
|
-
start_interactive
|
98
|
+
start_interactive
|
19
99
|
else
|
20
|
-
start_daemon
|
100
|
+
start_daemon
|
21
101
|
end
|
22
102
|
end
|
23
103
|
|
24
|
-
def
|
25
|
-
server = RabbitWQ::Server.new( options.merge( log: nil )
|
104
|
+
def start_interactive
|
105
|
+
server = RabbitWQ::Server.new( options.merge( log: nil ))
|
26
106
|
server.start
|
27
107
|
end
|
28
108
|
|
29
|
-
def
|
109
|
+
def start_daemon
|
30
110
|
server = RabbitWQ::ServerDaemon.new( options )
|
31
111
|
server.start
|
32
112
|
end
|
33
113
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
DEFAULT_PID_PATH = "/var/run/rabbit-wq/#{RabbitWQ::APP_ID}.pid"
|
39
|
-
|
40
|
-
global_opts = Trollop::options do
|
41
|
-
version RabbitWQ::VERSION_COPYRIGHT
|
42
|
-
banner <<-EOS
|
43
|
-
#{RabbitWQ::APP_NAME} #{RabbitWQ::VERSION_COPYRIGHT}
|
44
|
-
|
45
|
-
Usage:
|
46
|
-
#{RabbitWQ::APP_ID} [command] [options]
|
47
|
-
|
48
|
-
commands:
|
49
|
-
#{RabbitWQ::Cli::SUB_COMMANDS.map { |cmd| " #{cmd}" }.join( "\n" )}
|
50
|
-
|
51
|
-
(For help with a command: #{RabbitWQ::APP_ID} [command] -h)
|
114
|
+
def stop
|
115
|
+
server = RabbitWQ::ServerDaemon.new( options )
|
116
|
+
server.stop
|
117
|
+
end
|
52
118
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
end
|
119
|
+
def restart
|
120
|
+
stop
|
121
|
+
start_daemon
|
122
|
+
end
|
57
123
|
|
58
|
-
|
59
|
-
|
60
|
-
cmd = ARGV.shift || ''
|
61
|
-
cmd_opts = case( cmd )
|
62
|
-
#when "restart"
|
63
|
-
#Trollop::options do
|
64
|
-
#opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
65
|
-
#end
|
66
|
-
when "start"
|
67
|
-
Trollop::options do
|
68
|
-
opt :interactive, "Execute the server in interactive mode", :short => '-i'
|
69
|
-
opt :log_level, "The log level", :type => String, :default => 'info'
|
70
|
-
opt :log, "The path for the log file", :type => String, :short => '-l', :default => DEFAULT_LOG_PATH
|
71
|
-
opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
124
|
+
def status
|
125
|
+
RabbitWQ::ServerDaemon.new( options ).status
|
72
126
|
end
|
73
|
-
#when "status"
|
74
|
-
#Trollop::options do
|
75
|
-
#opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
76
|
-
#end
|
77
|
-
#when "stop"
|
78
|
-
#Trollop::options do
|
79
|
-
#opt :pid, "The path for the PID file", :type => String, :default => DEFAULT_PID_PATH
|
80
|
-
#end
|
81
|
-
else
|
82
|
-
Trollop::die "unknown command #{cmd.inspect}"
|
83
|
-
end
|
84
127
|
|
85
|
-
if cmd == 'start'
|
86
|
-
unless cmd_opts[:interactive]
|
87
|
-
Trollop::die( :log, "is required when running as daemon" ) unless cmd_opts[:log]
|
88
|
-
Trollop::die( :pid, "is required when running as daemon" ) unless cmd_opts[:pid]
|
89
128
|
end
|
90
129
|
end
|
91
|
-
|
92
|
-
if %w(restart status stop).include?( cmd )
|
93
|
-
Trollop::die( :pid, "is required" ) unless cmd_opts[:pid]
|
94
|
-
end
|
95
|
-
|
96
|
-
# Execute the command
|
97
|
-
#
|
98
|
-
case cmd
|
99
|
-
when "restart"
|
100
|
-
RabbitWQ::ServerDaemon.new( cmd_opts ).restart
|
101
|
-
when "start"
|
102
|
-
RabbitWQ::Cli.start cmd_opts
|
103
|
-
when "status"
|
104
|
-
RabbitWQ::ServerDaemon.new( cmd_opts ).status
|
105
|
-
when "stop"
|
106
|
-
RabbitWQ::ServerDaemon.new( cmd_opts ).stop
|
107
|
-
end
|
@@ -1,13 +1,67 @@
|
|
1
|
+
require 'oj'
|
2
|
+
|
1
3
|
module RabbitWQ
|
2
4
|
class Configuration
|
3
5
|
|
4
|
-
|
5
|
-
|
6
|
-
|
6
|
+
def self.attributes
|
7
|
+
%w(
|
8
|
+
delayed_exchange_prefix
|
9
|
+
delayed_queue_prefix
|
10
|
+
environment_file_path
|
11
|
+
env
|
12
|
+
error_queue
|
13
|
+
time_zone
|
14
|
+
work_exchange
|
15
|
+
work_queue
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
attr_accessor( *attributes )
|
20
|
+
|
21
|
+
def self.from_file( file_path )
|
22
|
+
options = Oj.load( File.read( file_path ))
|
23
|
+
RabbitWQ.configuration = Configuration.new
|
24
|
+
|
25
|
+
attributes.each do |c|
|
26
|
+
if options[c]
|
27
|
+
RabbitWQ.configuration.send( :"#{c}=", options[c] )
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def delayed_exchange_prefix
|
33
|
+
@delayed_exchange_prefix || 'work-delay'
|
34
|
+
end
|
35
|
+
|
36
|
+
def delayed_queue_prefix
|
37
|
+
@delayed_queue_prefix || 'work-delay'
|
38
|
+
end
|
39
|
+
|
40
|
+
def env
|
41
|
+
@env || 'production'
|
42
|
+
end
|
43
|
+
|
44
|
+
def error_queue
|
45
|
+
@error_queue || 'work-error'
|
46
|
+
end
|
47
|
+
|
48
|
+
def time_zone
|
49
|
+
@time_zone || 'UTC'
|
50
|
+
end
|
51
|
+
|
52
|
+
def work_exchange
|
53
|
+
@work_exchange || 'work'
|
54
|
+
end
|
55
|
+
|
56
|
+
def work_logger( log_level, path )
|
57
|
+
return if RabbitWQ.work_logger
|
58
|
+
|
59
|
+
RabbitWQ.work_logger = WorkLogger.new( log_level, path )
|
60
|
+
end
|
7
61
|
|
8
|
-
|
9
|
-
|
10
|
-
|
62
|
+
def work_queue
|
63
|
+
@work_queue || 'work'
|
64
|
+
end
|
11
65
|
|
12
66
|
end
|
13
67
|
end
|
data/lib/rabbit_wq/logging.rb
CHANGED
@@ -12,11 +12,19 @@ module RabbitWQ
|
|
12
12
|
).each do |level|
|
13
13
|
|
14
14
|
define_method level do |*messages|
|
15
|
+
return unless RabbitWQ.logger
|
15
16
|
messages.each do |message|
|
16
17
|
RabbitWQ.logger.send level, message
|
17
18
|
end
|
18
19
|
end
|
19
20
|
|
21
|
+
define_method "worker_#{level}" do |worker, *messages|
|
22
|
+
return unless RabbitWQ.work_logger
|
23
|
+
messages.each do |message|
|
24
|
+
RabbitWQ.work_logger.send level, worker, message
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
20
28
|
end
|
21
29
|
|
22
30
|
end
|
@@ -11,6 +11,8 @@ module RabbitWQ
|
|
11
11
|
REQUEUE = true
|
12
12
|
|
13
13
|
def call( options )
|
14
|
+
Time.zone = RabbitWQ.configuration.time_zone
|
15
|
+
|
14
16
|
channel = options[:channel]
|
15
17
|
delivery_info = options[:delivery_info]
|
16
18
|
metadata = options[:metadata]
|
@@ -23,15 +25,18 @@ module RabbitWQ
|
|
23
25
|
worker.call
|
24
26
|
channel.ack delivery_info.delivery_tag
|
25
27
|
rescue => e
|
26
|
-
|
27
|
-
handle_error( e, channel, delivery_info, payload, metadata )
|
28
|
+
handle_error( worker, e, channel, delivery_info, payload, metadata )
|
28
29
|
end
|
29
30
|
|
30
31
|
protected
|
31
32
|
|
32
|
-
def handle_error( e, channel, delivery_info, payload, metadata )
|
33
|
+
def handle_error( worker, e, channel, delivery_info, payload, metadata )
|
33
34
|
headers = metadata[:headers]
|
34
35
|
|
36
|
+
error_metadata = { type: e.class.name,
|
37
|
+
message: e.message,
|
38
|
+
backtrace: e.backtrace }
|
39
|
+
|
35
40
|
if headers['retry']
|
36
41
|
attempt = headers.fetch( 'attempt', 1 ).to_i
|
37
42
|
|
@@ -42,15 +47,15 @@ module RabbitWQ
|
|
42
47
|
retry_delay = retry_delays( attempt )
|
43
48
|
end
|
44
49
|
|
45
|
-
Work.enqueue_payload( payload, headers.merge( delay: retry_delay, attempt: attempt + 1 )
|
50
|
+
Work.enqueue_payload( payload, headers.merge( delay: retry_delay, attempt: attempt + 1 ).
|
51
|
+
merge( error: error_metadata ))
|
46
52
|
channel.nack delivery_info.delivery_tag
|
47
53
|
return
|
48
54
|
end
|
49
55
|
end
|
50
56
|
|
51
|
-
Work.enqueue_error_payload( payload, error:
|
52
|
-
|
53
|
-
backtrace: e.backtrace } )
|
57
|
+
Work.enqueue_error_payload( payload, error: error_metadata )
|
58
|
+
worker_error( worker, error_metadata.inspect )
|
54
59
|
channel.nack delivery_info.delivery_tag
|
55
60
|
return
|
56
61
|
end
|
data/lib/rabbit_wq/queues.rb
CHANGED
@@ -10,15 +10,17 @@ module RabbitWQ
|
|
10
10
|
end
|
11
11
|
|
12
12
|
def channel
|
13
|
-
@channel ||= mq.create_channel
|
13
|
+
@channel ||= mq.create_channel.tap do |c|
|
14
|
+
c.prefetch( 10 )
|
15
|
+
end
|
14
16
|
end
|
15
17
|
|
16
18
|
#def work_exchange
|
17
|
-
#@work_exchange ||= channel.direct(
|
19
|
+
#@work_exchange ||= channel.direct( RabbitWQ.configuration.work_exchange, durable: true )
|
18
20
|
#end
|
19
21
|
|
20
22
|
#def work_queue
|
21
|
-
#@work_queue ||= channel.queue(
|
23
|
+
#@work_queue ||= channel.queue( RabbitWQ.configuration.work_queue,
|
22
24
|
#durable: true ).
|
23
25
|
#bind( work_exchange )
|
24
26
|
#end
|
data/lib/rabbit_wq/server.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'bunny'
|
2
|
+
require 'celluloid/autostart'
|
2
3
|
|
3
4
|
module RabbitWQ
|
4
5
|
class Server
|
@@ -13,7 +14,6 @@ module RabbitWQ
|
|
13
14
|
|
14
15
|
def initialize( options )
|
15
16
|
@options = options
|
16
|
-
options[:pool_size] ||= 2 # TODO move to external configuration
|
17
17
|
|
18
18
|
configure_server
|
19
19
|
end
|
@@ -35,32 +35,73 @@ module RabbitWQ
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def work_exchange
|
38
|
-
@work_exchange ||= channel.direct(
|
38
|
+
@work_exchange ||= channel.direct( config.work_exchange, durable: true )
|
39
39
|
end
|
40
40
|
|
41
41
|
def work_queue
|
42
|
-
@work_queue ||= channel.queue(
|
42
|
+
@work_queue ||= channel.queue( config.work_queue,
|
43
43
|
durable: true ).
|
44
44
|
bind( work_exchange )
|
45
45
|
end
|
46
46
|
|
47
47
|
def pool
|
48
|
-
@pool ||= MessageHandler.pool( size: options[:
|
48
|
+
@pool ||= MessageHandler.pool( size: options[:threads] )
|
49
49
|
end
|
50
50
|
|
51
51
|
def run
|
52
|
+
if options[:threads] == 1
|
53
|
+
Celluloid::Actor[:message_handler] = MessageHandler.new
|
54
|
+
end
|
55
|
+
|
52
56
|
@work_consumer = work_queue.subscribe( manual_ack: true ) do |delivery_info, metadata, payload|
|
53
57
|
info "LISTENER RECEIVED #{payload}"
|
54
58
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
+
if options[:threads] > 1
|
60
|
+
pool.async.call( payload: payload,
|
61
|
+
delivery_info: delivery_info,
|
62
|
+
metadata: metadata,
|
63
|
+
channel: channel )
|
64
|
+
else
|
65
|
+
message_handler.call( payload: payload,
|
66
|
+
delivery_info: delivery_info,
|
67
|
+
metadata: metadata,
|
68
|
+
channel: channel )
|
69
|
+
end
|
59
70
|
end
|
60
71
|
end
|
61
72
|
|
73
|
+
def message_handler
|
74
|
+
Celluloid::Actor[:message_handler]
|
75
|
+
end
|
76
|
+
|
77
|
+
def config
|
78
|
+
RabbitWQ.configuration
|
79
|
+
end
|
80
|
+
|
62
81
|
def configure_server
|
82
|
+
load_configuration
|
63
83
|
initialize_loggers
|
84
|
+
load_environment
|
85
|
+
end
|
86
|
+
|
87
|
+
def load_configuration
|
88
|
+
if File.exists?( options[:config] )
|
89
|
+
options[:config_loaded] = true
|
90
|
+
Configuration.from_file( options[:config] )
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def load_environment
|
95
|
+
unless environment_file_path &&
|
96
|
+
File.exists?( environment_file_path )
|
97
|
+
return
|
98
|
+
end
|
99
|
+
|
100
|
+
require environment_file_path
|
101
|
+
end
|
102
|
+
|
103
|
+
def environment_file_path
|
104
|
+
RabbitWQ.configuration.environment_file_path
|
64
105
|
end
|
65
106
|
|
66
107
|
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
module RabbitWQ
|
6
|
+
class ServerDaemon
|
7
|
+
|
8
|
+
attr_reader :name,
|
9
|
+
:options,
|
10
|
+
:pid,
|
11
|
+
:pid_path,
|
12
|
+
:script,
|
13
|
+
:timeout
|
14
|
+
|
15
|
+
def initialize( options )
|
16
|
+
@options = options
|
17
|
+
@name = options[:name] || APP_NAME
|
18
|
+
@pid_path = options[:pid] || '.'
|
19
|
+
@pid = get_pid
|
20
|
+
@timeout = options[:timeout] || 10
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
abort "Process already running!" if process_exists?
|
25
|
+
|
26
|
+
pid = fork do
|
27
|
+
exit if fork
|
28
|
+
Process.setsid
|
29
|
+
exit if fork
|
30
|
+
store_pid( Process.pid )
|
31
|
+
File.umask 0000
|
32
|
+
redirect_output!
|
33
|
+
run
|
34
|
+
end
|
35
|
+
|
36
|
+
Process.waitpid( pid )
|
37
|
+
end
|
38
|
+
|
39
|
+
def run
|
40
|
+
Server.new( options ).start
|
41
|
+
end
|
42
|
+
|
43
|
+
def stop
|
44
|
+
kill_process
|
45
|
+
FileUtils.rm pid_path
|
46
|
+
end
|
47
|
+
|
48
|
+
def status
|
49
|
+
out = "#{APP_NAME} "
|
50
|
+
if process_exists?
|
51
|
+
out << "process running with PID: #{pid}"
|
52
|
+
else
|
53
|
+
out << "process does not exist"
|
54
|
+
end
|
55
|
+
$stdout.puts out
|
56
|
+
end
|
57
|
+
|
58
|
+
protected
|
59
|
+
|
60
|
+
#def create_pid( pid )
|
61
|
+
def store_pid( pid )
|
62
|
+
File.open( pid_path, 'w' ) do |f|
|
63
|
+
f.puts pid
|
64
|
+
end
|
65
|
+
rescue => e
|
66
|
+
$stderr.puts "Unable to open #{pid_path} for writing:\n\t(#{e.class}) #{e.message}"
|
67
|
+
exit!
|
68
|
+
end
|
69
|
+
|
70
|
+
def get_pid
|
71
|
+
return nil unless File.exists?( pid_path )
|
72
|
+
pid = nil
|
73
|
+
File.open( @pid_path, 'r' ) do |f|
|
74
|
+
pid = f.readline.to_s.gsub( /[^0-9]/, '' )
|
75
|
+
end
|
76
|
+
pid.to_i
|
77
|
+
rescue Errno::ENOENT
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
|
81
|
+
def remove_pidfile
|
82
|
+
File.unlink( pid_path )
|
83
|
+
rescue => e
|
84
|
+
$stderr.puts "Unable to unlink #{pid_path}:\n\t(#{e.class}) #{e.message}"
|
85
|
+
exit
|
86
|
+
end
|
87
|
+
|
88
|
+
def kill_process
|
89
|
+
abort "#{APP_NAME} process is not running" unless process_exists?
|
90
|
+
$stdout.write "Attempting to stop #{APP_NAME} process #{pid}..."
|
91
|
+
Process.kill INT, pid
|
92
|
+
iteration_num = 0
|
93
|
+
while process_exists? && iteration_num < 10
|
94
|
+
sleep 1
|
95
|
+
$stdout.write "."
|
96
|
+
iteration_num += 1
|
97
|
+
end
|
98
|
+
if process_exists?
|
99
|
+
$stderr.puts "\nFailed to stop #{APP_NAME} process #{pid}"
|
100
|
+
else
|
101
|
+
$stdout.puts "\nSuccessfuly stopped #{APP_NAME} process #{pid}"
|
102
|
+
end
|
103
|
+
rescue Errno::EPERM
|
104
|
+
$stderr.puts "No permission to query #{pid}!";
|
105
|
+
end
|
106
|
+
|
107
|
+
def process_exists?
|
108
|
+
return false unless pid
|
109
|
+
Process.kill( 0, pid )
|
110
|
+
true
|
111
|
+
rescue Errno::ESRCH, TypeError # "PID is NOT running or is zombied
|
112
|
+
false
|
113
|
+
rescue Errno::EPERM
|
114
|
+
$stderr.puts "No permission to query #{pid}!";
|
115
|
+
false
|
116
|
+
end
|
117
|
+
|
118
|
+
def redirect_output!
|
119
|
+
if log_path = options[:log]
|
120
|
+
#puts "redirecting to log"
|
121
|
+
# if the log directory doesn't exist, create it
|
122
|
+
FileUtils.mkdir_p( File.dirname( log_path ), :mode => 0755 )
|
123
|
+
# touch the log file to create it
|
124
|
+
FileUtils.touch( log_path )
|
125
|
+
# Set permissions on the log file
|
126
|
+
File.chmod( 0644, log_path )
|
127
|
+
# Reopen $stdout (NOT +STDOUT+) to start writing to the log file
|
128
|
+
$stdout.reopen( log_path, 'a' )
|
129
|
+
# Redirect $stderr to $stdout
|
130
|
+
$stderr.reopen $stdout
|
131
|
+
$stdout.sync = true
|
132
|
+
else
|
133
|
+
#puts "redirecting to /dev/null"
|
134
|
+
# We're not bothering to sync if we're dumping to /dev/null
|
135
|
+
# because /dev/null doesn't care about buffered output
|
136
|
+
$stdin.reopen '/dev/null'
|
137
|
+
$stdout.reopen '/dev/null', 'a'
|
138
|
+
$stderr.reopen $stdout
|
139
|
+
end
|
140
|
+
log_path = options[:log] ? options[:log] : '/dev/null'
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
end
|
@@ -8,23 +8,16 @@ module RabbitWQ
|
|
8
8
|
protected
|
9
9
|
|
10
10
|
def initialize_loggers
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
l.adapter $stderr, :level => [:error, :fatal]
|
16
|
-
end
|
17
|
-
else
|
18
|
-
RabbitWQ.logger = Yell.new do |l|
|
19
|
-
l.level = log_level
|
20
|
-
l.adapter :file, options[:log]
|
21
|
-
end
|
22
|
-
|
23
|
-
Celluloid.logger = Yell.new do |l|
|
24
|
-
l.level = :info
|
25
|
-
l.adapter :file, File.join( File.dirname( options[:log] ), "#{APP_ID}-celluloid.log" )
|
26
|
-
end
|
11
|
+
RabbitWQ.logger = Yell.new do |l|
|
12
|
+
l.level = log_level
|
13
|
+
l.adapter $stdout, :level => [:debug, :info, :warn]
|
14
|
+
l.adapter $stderr, :level => [:error, :fatal]
|
27
15
|
end
|
16
|
+
|
17
|
+
#Celluloid.logger = Yell.new do |l|
|
18
|
+
#l.level = :info
|
19
|
+
#l.adapter :file, File.join( File.dirname( options[:log] ), "#{APP_ID}-celluloid.log" )
|
20
|
+
#end
|
28
21
|
end
|
29
22
|
|
30
23
|
def log_startup
|
@@ -37,12 +30,16 @@ module RabbitWQ
|
|
37
30
|
[
|
38
31
|
"",
|
39
32
|
"***",
|
40
|
-
"* #{
|
33
|
+
"* #{APP_NAME} started",
|
41
34
|
"*",
|
42
35
|
"* #{VERSION_COPYRIGHT}",
|
36
|
+
"*",
|
37
|
+
"* Configuration:",
|
38
|
+
(options[:config_loaded] ? "* file: #{options[:config]}" : nil),
|
39
|
+
RabbitWQ::Configuration.attributes.map { |a| "* #{a}: #{RabbitWQ.configuration.send( a )}" },
|
40
|
+
"*",
|
43
41
|
"***",
|
44
|
-
|
45
|
-
]
|
42
|
+
].flatten.reject( &:nil? )
|
46
43
|
end
|
47
44
|
|
48
45
|
def log_level
|
data/lib/rabbit_wq/version.rb
CHANGED
data/lib/rabbit_wq/work.rb
CHANGED
@@ -12,37 +12,80 @@ module RabbitWQ
|
|
12
12
|
delay = options.delete( :delay )
|
13
13
|
delay = nil if delay && delay < 5000
|
14
14
|
|
15
|
-
mq = ::Bunny.new.tap { |bunny| bunny.start }
|
16
|
-
channel = mq.create_channel
|
17
|
-
|
18
15
|
if delay
|
19
|
-
|
20
|
-
|
16
|
+
with_channel do |channel|
|
17
|
+
delay_x = channel.direct( "#{RabbitWQ.configuration.delayed_exchange_prefix}-#{delay}ms", durable: true )
|
18
|
+
work_x = channel.direct( RabbitWQ.configuration.work_exchange, durable: true )
|
21
19
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
20
|
+
channel.queue( "#{RabbitWQ.configuration.delayed_queue_prefix}-#{delay}ms",
|
21
|
+
durable: true,
|
22
|
+
arguments: { "x-dead-letter-exchange" => work_x.name,
|
23
|
+
"x-message-ttl" => delay } ).
|
24
|
+
bind( delay_x )
|
25
|
+
|
26
|
+
delay_x.publish( payload, durable: true,
|
27
|
+
content_type: 'application/yaml',
|
28
|
+
headers: options )
|
29
|
+
end
|
27
30
|
|
28
|
-
delay_x.publish( payload, durable: true,
|
29
|
-
headers: options )
|
30
31
|
return
|
31
32
|
end
|
32
33
|
|
33
|
-
|
34
|
-
|
35
|
-
|
34
|
+
with_work_exchange do |work_x, work_q|
|
35
|
+
work_x.publish( payload, durable: true,
|
36
|
+
content_type: 'application/yaml',
|
37
|
+
headers: options )
|
38
|
+
end
|
36
39
|
end
|
37
40
|
|
38
41
|
def self.enqueue_error_payload( payload, options={} )
|
39
|
-
|
40
|
-
|
42
|
+
with_channel do |channel|
|
43
|
+
error_q = channel.queue( RabbitWQ.configuration.error_queue, durable: true )
|
44
|
+
error_q.publish( payload, durable: true,
|
45
|
+
content_type: 'application/yaml',
|
46
|
+
headers: options )
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.with_work_exchange
|
51
|
+
with_channel do |channel|
|
52
|
+
begin
|
53
|
+
exchange = channel.direct( RabbitWQ.configuration.work_exchange, durable: true )
|
54
|
+
channel.queue( RabbitWQ.configuration.work_queue, durable: true ).tap do |q|
|
55
|
+
q.bind( exchange )
|
56
|
+
yield exchange, q
|
57
|
+
end
|
58
|
+
ensure
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
41
62
|
|
42
|
-
|
43
|
-
|
44
|
-
|
63
|
+
def self.with_channel
|
64
|
+
Bunny.new.tap do |b|
|
65
|
+
b.start
|
66
|
+
begin
|
67
|
+
b.create_channel.tap do |c|
|
68
|
+
yield c
|
69
|
+
end
|
70
|
+
ensure
|
71
|
+
b.stop
|
72
|
+
end
|
73
|
+
end
|
45
74
|
end
|
46
75
|
|
76
|
+
#def self.with_exchange
|
77
|
+
#Bunny.new.tap do |b|
|
78
|
+
#b.start
|
79
|
+
#begin
|
80
|
+
#b.create_channel.tap do |c|
|
81
|
+
#queue = c.queue( 'replication', durable: true )
|
82
|
+
#yield c.default_exchange, queue.name
|
83
|
+
#end
|
84
|
+
#ensure
|
85
|
+
#b.stop
|
86
|
+
#end
|
87
|
+
#end
|
88
|
+
#end
|
89
|
+
|
47
90
|
end
|
48
91
|
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'ansi'
|
2
|
+
require 'yell'
|
3
|
+
|
4
|
+
module RabbitWQ
|
5
|
+
class WorkLogger
|
6
|
+
|
7
|
+
def initialize( log_level, path )
|
8
|
+
@logger = Yell.new do |l|
|
9
|
+
l.level = log_level
|
10
|
+
l.adapter :file, path
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
%w(
|
15
|
+
debug
|
16
|
+
error
|
17
|
+
fatal
|
18
|
+
info
|
19
|
+
warn
|
20
|
+
).each do |level|
|
21
|
+
|
22
|
+
define_method level do |*args|
|
23
|
+
worker, message = nil, nil
|
24
|
+
if args.size > 1
|
25
|
+
worker, message = *args
|
26
|
+
else
|
27
|
+
message = args.first
|
28
|
+
end
|
29
|
+
|
30
|
+
if worker
|
31
|
+
logger.send( level, "[" + ANSI.cyan { "#{worker.class.name}:#{worker.object_id}" } + "] #{message}" )
|
32
|
+
else
|
33
|
+
logger.send( level, message )
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
def level=( l )
|
40
|
+
end
|
41
|
+
|
42
|
+
protected
|
43
|
+
|
44
|
+
attr_reader :logger
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
data/lib/rabbit_wq/worker.rb
CHANGED
@@ -1,6 +1,27 @@
|
|
1
1
|
module RabbitWQ
|
2
2
|
module Worker
|
3
3
|
|
4
|
+
def self.included( other_module )
|
5
|
+
other_module.class_eval do
|
6
|
+
%w(
|
7
|
+
debug
|
8
|
+
error
|
9
|
+
fatal
|
10
|
+
info
|
11
|
+
warn
|
12
|
+
).each do |level|
|
13
|
+
|
14
|
+
define_method level do |*messages|
|
15
|
+
return unless RabbitWQ.logger
|
16
|
+
messages.each do |message|
|
17
|
+
RabbitWQ.work_logger.send level, self, message
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
4
25
|
def work( options={} )
|
5
26
|
RabbitWQ::Work.enqueue( self, options )
|
6
27
|
self
|
data/rabbit-wq.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rabbit-wq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-
|
12
|
+
date: 2013-12-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -75,6 +75,22 @@ dependencies:
|
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
77
|
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: oj
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ~>
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '2'
|
86
|
+
type: :runtime
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ~>
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '2'
|
78
94
|
- !ruby/object:Gem::Dependency
|
79
95
|
name: trollop
|
80
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -118,11 +134,11 @@ files:
|
|
118
134
|
- .gitignore
|
119
135
|
- .ruby-gemset
|
120
136
|
- .ruby-version
|
121
|
-
- Gemfile
|
122
137
|
- LICENSE.txt
|
123
138
|
- README.md
|
124
139
|
- Rakefile
|
125
140
|
- bin/rabbit-wq
|
141
|
+
- lib/ansi.rb
|
126
142
|
- lib/rabbit-wq.rb
|
127
143
|
- lib/rabbit_wq.rb
|
128
144
|
- lib/rabbit_wq/cli.rb
|
@@ -132,9 +148,11 @@ files:
|
|
132
148
|
- lib/rabbit_wq/message_handler.rb
|
133
149
|
- lib/rabbit_wq/queues.rb
|
134
150
|
- lib/rabbit_wq/server.rb
|
151
|
+
- lib/rabbit_wq/server_daemon.rb
|
135
152
|
- lib/rabbit_wq/server_logging.rb
|
136
153
|
- lib/rabbit_wq/version.rb
|
137
154
|
- lib/rabbit_wq/work.rb
|
155
|
+
- lib/rabbit_wq/work_logger.rb
|
138
156
|
- lib/rabbit_wq/worker.rb
|
139
157
|
- rabbit-wq.gemspec
|
140
158
|
homepage: ''
|
@@ -150,18 +168,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
150
168
|
- - ! '>='
|
151
169
|
- !ruby/object:Gem::Version
|
152
170
|
version: '0'
|
153
|
-
segments:
|
154
|
-
- 0
|
155
|
-
hash: -1504652558847638442
|
156
171
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
157
172
|
none: false
|
158
173
|
requirements:
|
159
174
|
- - ! '>='
|
160
175
|
- !ruby/object:Gem::Version
|
161
176
|
version: '0'
|
162
|
-
segments:
|
163
|
-
- 0
|
164
|
-
hash: -1504652558847638442
|
165
177
|
requirements: []
|
166
178
|
rubyforge_project:
|
167
179
|
rubygems_version: 1.8.25
|
@@ -169,3 +181,4 @@ signing_key:
|
|
169
181
|
specification_version: 3
|
170
182
|
summary: A work queue built on RabbitMQ and Celluloid.
|
171
183
|
test_files: []
|
184
|
+
has_rdoc:
|
data/Gemfile
DELETED