queue_map 0.4 → 0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/queue_map_consumer +7 -5
- data/lib/queue_map/consumer.rb +46 -46
- data/lib/queue_map/version.rb +1 -1
- data/lib/queue_map.rb +16 -8
- metadata +4 -4
data/bin/queue_map_consumer
CHANGED
@@ -46,19 +46,21 @@ def start
|
|
46
46
|
puts "Starting consumers..."
|
47
47
|
unless @options[:foreground]
|
48
48
|
# daemonize
|
49
|
-
puts "Daemonizing"
|
50
|
-
|
51
|
-
|
49
|
+
puts "Daemonizing..."
|
50
|
+
exit if Kernel.fork
|
51
|
+
Process.setsid
|
52
|
+
exit if Kernel.fork
|
53
|
+
|
54
|
+
if pid = Kernel.fork
|
52
55
|
Process.detach(pid)
|
53
56
|
puts "Daemonized as #{pid}"
|
54
57
|
File.open(@consumer.pid_file, "w") { |f| f << pid.to_s }
|
55
58
|
exit 0
|
56
59
|
else
|
57
|
-
Process.setsid
|
58
|
-
exit if fork
|
59
60
|
f = File.open(@consumer.log_file, "a")
|
60
61
|
STDOUT.reopen(f)
|
61
62
|
STDERR.reopen(f)
|
63
|
+
STDIN.close
|
62
64
|
end
|
63
65
|
end
|
64
66
|
@consumer.start
|
data/lib/queue_map/consumer.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
class QueueMap::Consumer
|
2
|
-
attr_accessor :count_workers, :worker_proc, :on_exception_proc
|
2
|
+
attr_accessor :count_workers, :worker_proc, :on_exception_proc, :job_timeout
|
3
3
|
attr_reader :name, :master_pid
|
4
|
-
attr_writer :pid_file, :log_file
|
4
|
+
attr_writer :pid_file, :log_file, :idle_proc
|
5
5
|
class Configurator
|
6
6
|
def initialize(base)
|
7
7
|
@base = base
|
@@ -9,12 +9,15 @@ class QueueMap::Consumer
|
|
9
9
|
|
10
10
|
def before_fork(&block); @base.before_fork_procs << block; end
|
11
11
|
def after_fork(&block); @base.after_fork_procs << block; end
|
12
|
-
def
|
12
|
+
def after_response(&block); @base.after_response_procs << block; end
|
13
|
+
def before_job(&block); @base.before_job_procs << block; end
|
13
14
|
def worker(&block); @base.worker_proc = block; end
|
14
15
|
def on_exception(&block); @base.on_exception_proc = block; end
|
15
16
|
def count_workers(value); @base.count_workers = value; end
|
16
17
|
def pid_file(value); @base.pid_file = value; end
|
17
18
|
def log_file(value); @base.log_file = value; end
|
19
|
+
def idle(&block); @base.idle_proc = block; end
|
20
|
+
def job_timeout(value); @base.job_timeout = value; end
|
18
21
|
|
19
22
|
def respond_to?(*args)
|
20
23
|
super || @base.respond_to?(*args)
|
@@ -44,8 +47,6 @@ class QueueMap::Consumer
|
|
44
47
|
case options[:strategy]
|
45
48
|
when :fork
|
46
49
|
extend(ForkStrategy)
|
47
|
-
when :thread
|
48
|
-
extend(ThreadStrategy)
|
49
50
|
when :test
|
50
51
|
nil
|
51
52
|
else
|
@@ -61,8 +62,12 @@ class QueueMap::Consumer
|
|
61
62
|
@before_fork_procs ||= []
|
62
63
|
end
|
63
64
|
|
64
|
-
def
|
65
|
-
@
|
65
|
+
def after_response_procs
|
66
|
+
@after_response_procs ||= []
|
67
|
+
end
|
68
|
+
|
69
|
+
def before_job_procs
|
70
|
+
@before_job_procs ||= []
|
66
71
|
end
|
67
72
|
|
68
73
|
def pid_file
|
@@ -85,51 +90,46 @@ class QueueMap::Consumer
|
|
85
90
|
raise RuntimeError, "Called stop on Consumer without strategy"
|
86
91
|
end
|
87
92
|
|
93
|
+
def idle_proc
|
94
|
+
@idle_proc ||= lambda { sleep 0.05 }
|
95
|
+
end
|
96
|
+
|
88
97
|
def run_consumer
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
#
|
93
|
-
return if @shutting_down
|
98
|
+
begin
|
99
|
+
QueueMap.with_bunny do |bunny|
|
100
|
+
q = bunny.queue(name.to_s, :durable => false, :auto_delete => false, :ack => true)
|
101
|
+
logger.info "Process #{Process.pid} is listening on #{name.to_s}"
|
94
102
|
begin
|
95
|
-
|
96
|
-
msg
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
103
|
+
msg = q.pop
|
104
|
+
(idle_proc.call; next) if msg == :queue_empty
|
105
|
+
before_job_procs.each { |p| p.call }
|
106
|
+
begin
|
107
|
+
Timeout.timeout(job_timeout) do
|
108
|
+
msg = Marshal.load(msg)
|
109
|
+
logger.info { "Received #{msg.inspect} on #{Process.pid}" }
|
110
|
+
result = worker_proc.call(msg[:input])
|
111
|
+
logger.info { "Result #{result.inspect}" }
|
112
|
+
bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
|
113
|
+
end
|
114
|
+
ensure
|
115
|
+
after_response_procs.each { |p| p.call }
|
106
116
|
end
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
module ThreadStrategy
|
113
|
-
def start
|
114
|
-
@threads = []
|
115
|
-
count_workers.times do |c|
|
116
|
-
@threads << (Thread.new do
|
117
|
-
begin
|
118
|
-
run_consumer
|
119
|
-
rescue Exception => e
|
120
|
-
logger.error %(#{e}\n#{e.backtrace.join("\n")})
|
121
|
-
end
|
122
|
-
end)
|
117
|
+
rescue Qrack::ClientTimeout
|
118
|
+
rescue Timeout::Error
|
119
|
+
logger.info "Job took longer than #{timeout} seconds to complete. Aborting"
|
120
|
+
end while ! @shutting_down
|
123
121
|
end
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
if graceful
|
128
|
-
@shutting_down = true
|
122
|
+
rescue Exception => e # Bunny gets into a strange state when exceptions are raised, so reconnect to queue server if it happens
|
123
|
+
if on_exception_proc
|
124
|
+
on_exception_proc.call(e)
|
129
125
|
else
|
130
|
-
|
126
|
+
logger.info e.class
|
127
|
+
logger.error e.message
|
128
|
+
logger.error e.backtrace
|
131
129
|
end
|
132
|
-
|
130
|
+
sleep 0.2
|
131
|
+
end while ! @shutting_down
|
132
|
+
logger.info "Done."
|
133
133
|
end
|
134
134
|
|
135
135
|
module ForkStrategy
|
data/lib/queue_map/version.rb
CHANGED
data/lib/queue_map.rb
CHANGED
@@ -3,8 +3,9 @@ require "bunny"
|
|
3
3
|
require 'timeout'
|
4
4
|
|
5
5
|
module QueueMap
|
6
|
-
autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
|
7
6
|
BUNNY_MUTEX = Mutex.new
|
7
|
+
|
8
|
+
autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
|
8
9
|
extend self
|
9
10
|
attr_accessor :mode, :consumer_path
|
10
11
|
attr_writer :consumer_base_path
|
@@ -41,15 +42,15 @@ module QueueMap
|
|
41
42
|
response_queue = bunny.queue(response_queue_name, :durable => false, :exclusive => true, :auto_delete => true)
|
42
43
|
|
43
44
|
(0..(collection.length - 1)).each do |i|
|
45
|
+
puts "Publishing #{collection[i].inspect}."
|
44
46
|
q.publish(Marshal.dump(:input => collection[i], :index => i, :response_queue => response_queue_name))
|
45
47
|
end
|
46
48
|
|
47
49
|
results = {}
|
48
50
|
begin
|
49
51
|
Timeout.timeout(options[:timeout] || 5) do
|
50
|
-
collection.length
|
51
|
-
|
52
|
-
response = Marshal.load(next_response)
|
52
|
+
response_queue.subscribe(:message_max => collection.length) do |msg|
|
53
|
+
response = Marshal.load(msg)
|
53
54
|
results[response[:index]] = response[:result]
|
54
55
|
end
|
55
56
|
end
|
@@ -71,19 +72,26 @@ module QueueMap
|
|
71
72
|
end
|
72
73
|
|
73
74
|
def consumer(name)
|
74
|
-
consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :
|
75
|
+
consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :fork)
|
75
76
|
end
|
76
77
|
|
77
|
-
def
|
78
|
-
bunny = nil
|
78
|
+
def new_bunny_connection
|
79
79
|
BUNNY_MUTEX.synchronize do
|
80
80
|
bunny = Bunny.new((@connection_info || { }).merge(:spec => '08'))
|
81
81
|
bunny.start
|
82
|
+
bunny
|
82
83
|
end
|
84
|
+
end
|
85
|
+
|
86
|
+
def with_bunny(&block)
|
87
|
+
bunny = new_bunny_connection
|
83
88
|
begin
|
84
89
|
yield bunny
|
85
90
|
ensure
|
86
|
-
|
91
|
+
BUNNY_MUTEX.synchronize do
|
92
|
+
bunny.stop rescue nil
|
93
|
+
bunny.close_connection rescue nil
|
94
|
+
end
|
87
95
|
end
|
88
96
|
end
|
89
97
|
|
metadata
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: queue_map
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 1
|
5
5
|
prerelease: false
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
version: "0.
|
8
|
+
- 5
|
9
|
+
version: "0.5"
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Tim Harper
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-10-
|
17
|
+
date: 2010-10-14 00:00:00 -06:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|