queue_map 0.4 → 0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -46,19 +46,21 @@ def start
46
46
  puts "Starting consumers..."
47
47
  unless @options[:foreground]
48
48
  # daemonize
49
- puts "Daemonizing"
50
- pid = Kernel.fork
51
- if pid
49
+ puts "Daemonizing..."
50
+ exit if Kernel.fork
51
+ Process.setsid
52
+ exit if Kernel.fork
53
+
54
+ if pid = Kernel.fork
52
55
  Process.detach(pid)
53
56
  puts "Daemonized as #{pid}"
54
57
  File.open(@consumer.pid_file, "w") { |f| f << pid.to_s }
55
58
  exit 0
56
59
  else
57
- Process.setsid
58
- exit if fork
59
60
  f = File.open(@consumer.log_file, "a")
60
61
  STDOUT.reopen(f)
61
62
  STDERR.reopen(f)
63
+ STDIN.close
62
64
  end
63
65
  end
64
66
  @consumer.start
@@ -1,7 +1,7 @@
1
1
  class QueueMap::Consumer
2
- attr_accessor :count_workers, :worker_proc, :on_exception_proc
2
+ attr_accessor :count_workers, :worker_proc, :on_exception_proc, :job_timeout
3
3
  attr_reader :name, :master_pid
4
- attr_writer :pid_file, :log_file
4
+ attr_writer :pid_file, :log_file, :idle_proc
5
5
  class Configurator
6
6
  def initialize(base)
7
7
  @base = base
@@ -9,12 +9,15 @@ class QueueMap::Consumer
9
9
 
10
10
  def before_fork(&block); @base.before_fork_procs << block; end
11
11
  def after_fork(&block); @base.after_fork_procs << block; end
12
- def between_responses(&block); @base.between_responses_procs << block; end
12
+ def after_response(&block); @base.after_response_procs << block; end
13
+ def before_job(&block); @base.before_job_procs << block; end
13
14
  def worker(&block); @base.worker_proc = block; end
14
15
  def on_exception(&block); @base.on_exception_proc = block; end
15
16
  def count_workers(value); @base.count_workers = value; end
16
17
  def pid_file(value); @base.pid_file = value; end
17
18
  def log_file(value); @base.log_file = value; end
19
+ def idle(&block); @base.idle_proc = block; end
20
+ def job_timeout(value); @base.job_timeout = value; end
18
21
 
19
22
  def respond_to?(*args)
20
23
  super || @base.respond_to?(*args)
@@ -44,8 +47,6 @@ class QueueMap::Consumer
44
47
  case options[:strategy]
45
48
  when :fork
46
49
  extend(ForkStrategy)
47
- when :thread
48
- extend(ThreadStrategy)
49
50
  when :test
50
51
  nil
51
52
  else
@@ -61,8 +62,12 @@ class QueueMap::Consumer
61
62
  @before_fork_procs ||= []
62
63
  end
63
64
 
64
- def between_responses_procs
65
- @between_responses_procs ||= []
65
+ def after_response_procs
66
+ @after_response_procs ||= []
67
+ end
68
+
69
+ def before_job_procs
70
+ @before_job_procs ||= []
66
71
  end
67
72
 
68
73
  def pid_file
@@ -85,51 +90,46 @@ class QueueMap::Consumer
85
90
  raise RuntimeError, "Called stop on Consumer without strategy"
86
91
  end
87
92
 
93
+ def idle_proc
94
+ @idle_proc ||= lambda { sleep 0.05 }
95
+ end
96
+
88
97
  def run_consumer
89
- QueueMap.with_bunny do |bunny|
90
- q = bunny.queue(name.to_s, :durable => false, :auto_delete => true)
91
- while msg = q.pop
92
- # STDOUT << "[#{Thread.current[:id]}]"
93
- return if @shutting_down
98
+ begin
99
+ QueueMap.with_bunny do |bunny|
100
+ q = bunny.queue(name.to_s, :durable => false, :auto_delete => false, :ack => true)
101
+ logger.info "Process #{Process.pid} is listening on #{name.to_s}"
94
102
  begin
95
- (sleep 0.05; next) if msg == :queue_empty
96
- msg = Marshal.load(msg)
97
- result = worker_proc.call(msg[:input])
98
- bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
99
- between_responses_procs.each { |p| p.call }
100
- rescue Exception => e
101
- if on_exception_proc
102
- on_exception_proc.call(e)
103
- else
104
- logger.error e.message
105
- logger.error e.backtrace
103
+ msg = q.pop
104
+ (idle_proc.call; next) if msg == :queue_empty
105
+ before_job_procs.each { |p| p.call }
106
+ begin
107
+ Timeout.timeout(job_timeout) do
108
+ msg = Marshal.load(msg)
109
+ logger.info { "Received #{msg.inspect} on #{Process.pid}" }
110
+ result = worker_proc.call(msg[:input])
111
+ logger.info { "Result #{result.inspect}" }
112
+ bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
113
+ end
114
+ ensure
115
+ after_response_procs.each { |p| p.call }
106
116
  end
107
- end
108
- end
109
- end
110
- end
111
-
112
- module ThreadStrategy
113
- def start
114
- @threads = []
115
- count_workers.times do |c|
116
- @threads << (Thread.new do
117
- begin
118
- run_consumer
119
- rescue Exception => e
120
- logger.error %(#{e}\n#{e.backtrace.join("\n")})
121
- end
122
- end)
117
+ rescue Qrack::ClientTimeout
118
+ rescue Timeout::Error
119
+ logger.info "Job took longer than #{timeout} seconds to complete. Aborting"
120
+ end while ! @shutting_down
123
121
  end
124
- end
125
-
126
- def stop(graceful = true)
127
- if graceful
128
- @shutting_down = true
122
+ rescue Exception => e # Bunny gets into a strange state when exceptions are raised, so reconnect to queue server if it happens
123
+ if on_exception_proc
124
+ on_exception_proc.call(e)
129
125
  else
130
- @threads.each { |t| t.kill }
126
+ logger.info e.class
127
+ logger.error e.message
128
+ logger.error e.backtrace
131
129
  end
132
- end
130
+ sleep 0.2
131
+ end while ! @shutting_down
132
+ logger.info "Done."
133
133
  end
134
134
 
135
135
  module ForkStrategy
@@ -1,3 +1,3 @@
1
1
  module QueueMap
2
- VERSION="0.4"
2
+ VERSION="0.5"
3
3
  end
data/lib/queue_map.rb CHANGED
@@ -3,8 +3,9 @@ require "bunny"
3
3
  require 'timeout'
4
4
 
5
5
  module QueueMap
6
- autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
7
6
  BUNNY_MUTEX = Mutex.new
7
+
8
+ autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
8
9
  extend self
9
10
  attr_accessor :mode, :consumer_path
10
11
  attr_writer :consumer_base_path
@@ -41,15 +42,15 @@ module QueueMap
41
42
  response_queue = bunny.queue(response_queue_name, :durable => false, :exclusive => true, :auto_delete => true)
42
43
 
43
44
  (0..(collection.length - 1)).each do |i|
45
+ puts "Publishing #{collection[i].inspect}."
44
46
  q.publish(Marshal.dump(:input => collection[i], :index => i, :response_queue => response_queue_name))
45
47
  end
46
48
 
47
49
  results = {}
48
50
  begin
49
51
  Timeout.timeout(options[:timeout] || 5) do
50
- collection.length.times do
51
- sleep 0.05 while (next_response = response_queue.pop) == :queue_empty
52
- response = Marshal.load(next_response)
52
+ response_queue.subscribe(:message_max => collection.length) do |msg|
53
+ response = Marshal.load(msg)
53
54
  results[response[:index]] = response[:result]
54
55
  end
55
56
  end
@@ -71,19 +72,26 @@ module QueueMap
71
72
  end
72
73
 
73
74
  def consumer(name)
74
- consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :thread)
75
+ consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :fork)
75
76
  end
76
77
 
77
- def with_bunny(&block)
78
- bunny = nil
78
+ def new_bunny_connection
79
79
  BUNNY_MUTEX.synchronize do
80
80
  bunny = Bunny.new((@connection_info || { }).merge(:spec => '08'))
81
81
  bunny.start
82
+ bunny
82
83
  end
84
+ end
85
+
86
+ def with_bunny(&block)
87
+ bunny = new_bunny_connection
83
88
  begin
84
89
  yield bunny
85
90
  ensure
86
- (bunny.close_connection unless bunny.status == :not_connected) rescue nil
91
+ BUNNY_MUTEX.synchronize do
92
+ bunny.stop rescue nil
93
+ bunny.close_connection rescue nil
94
+ end
87
95
  end
88
96
  end
89
97
 
metadata CHANGED
@@ -1,12 +1,12 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: queue_map
3
3
  version: !ruby/object:Gem::Version
4
- hash: 3
4
+ hash: 1
5
5
  prerelease: false
6
6
  segments:
7
7
  - 0
8
- - 4
9
- version: "0.4"
8
+ - 5
9
+ version: "0.5"
10
10
  platform: ruby
11
11
  authors:
12
12
  - Tim Harper
@@ -14,7 +14,7 @@ autorequire:
14
14
  bindir: bin
15
15
  cert_chain: []
16
16
 
17
- date: 2010-10-12 00:00:00 -06:00
17
+ date: 2010-10-14 00:00:00 -06:00
18
18
  default_executable:
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency