queue_map 0.4 → 0.5

data/bin/queue_map_consumer

@@ -46,19 +46,21 @@ def start
   puts "Starting consumers..."
   unless @options[:foreground]
     # daemonize
-    puts "Daemonizing"
-    pid = Kernel.fork
-    if pid
+    puts "Daemonizing..."
+    exit if Kernel.fork
+    Process.setsid
+    exit if Kernel.fork
+
+    if pid = Kernel.fork
       Process.detach(pid)
       puts "Daemonized as #{pid}"
       File.open(@consumer.pid_file, "w") { |f| f << pid.to_s }
       exit 0
     else
-      Process.setsid
-      exit if fork
       f = File.open(@consumer.log_file, "a")
       STDOUT.reopen(f)
       STDERR.reopen(f)
+      STDIN.close
     end
   end
   @consumer.start

data/lib/queue_map/consumer.rb

@@ -1,7 +1,7 @@
 class QueueMap::Consumer
-  attr_accessor :count_workers, :worker_proc, :on_exception_proc
+  attr_accessor :count_workers, :worker_proc, :on_exception_proc, :job_timeout
   attr_reader :name, :master_pid
-  attr_writer :pid_file, :log_file
+  attr_writer :pid_file, :log_file, :idle_proc
   class Configurator
     def initialize(base)
       @base = base
@@ -9,12 +9,15 @@ class QueueMap::Consumer
 
     def before_fork(&block);        @base.before_fork_procs       << block; end
     def after_fork(&block);         @base.after_fork_procs        << block; end
-    def between_responses(&block);  @base.between_responses_procs << block; end
+    def after_response(&block);     @base.after_response_procs    << block; end
+    def before_job(&block);         @base.before_job_procs        << block; end
     def worker(&block);             @base.worker_proc             =  block; end
     def on_exception(&block);       @base.on_exception_proc       =  block; end
     def count_workers(value);       @base.count_workers           =  value; end
     def pid_file(value);            @base.pid_file                =  value; end
     def log_file(value);            @base.log_file                =  value; end
+    def idle(&block);               @base.idle_proc               =  block; end
+    def job_timeout(value);         @base.job_timeout             =  value; end
 
     def respond_to?(*args)
       super || @base.respond_to?(*args)
@@ -44,8 +47,6 @@ class QueueMap::Consumer
     case options[:strategy]
     when :fork
       extend(ForkStrategy)
-    when :thread
-      extend(ThreadStrategy)
     when :test
       nil
     else
@@ -61,8 +62,12 @@ class QueueMap::Consumer
     @before_fork_procs ||= []
   end
 
-  def between_responses_procs
-    @between_responses_procs ||= []
+  def after_response_procs
+    @after_response_procs ||= []
+  end
+
+  def before_job_procs
+    @before_job_procs ||= []
   end
   
   def pid_file
@@ -85,51 +90,46 @@ class QueueMap::Consumer
     raise RuntimeError, "Called stop on Consumer without strategy"
   end
 
+  def idle_proc
+    @idle_proc ||= lambda { sleep 0.05 }
+  end
+
   def run_consumer
-    QueueMap.with_bunny do |bunny|
-      q = bunny.queue(name.to_s, :durable => false, :auto_delete => true)
-      while msg = q.pop
-        # STDOUT << "[#{Thread.current[:id]}]"
-        return if @shutting_down
+    begin
+      QueueMap.with_bunny do |bunny|
+        q = bunny.queue(name.to_s, :durable => false, :auto_delete => false, :ack => true)
+        logger.info "Process #{Process.pid} is listening on #{name.to_s}"
         begin
-          (sleep 0.05; next) if msg == :queue_empty
-          msg = Marshal.load(msg)
-          result = worker_proc.call(msg[:input])
-          bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
-          between_responses_procs.each { |p| p.call }
-        rescue Exception => e
-          if on_exception_proc
-            on_exception_proc.call(e)
-          else
-            logger.error e.message
-            logger.error e.backtrace
+          msg = q.pop
+          (idle_proc.call; next) if msg == :queue_empty
+          before_job_procs.each { |p| p.call }
+          begin
+            Timeout.timeout(job_timeout) do
+              msg = Marshal.load(msg)
+              logger.info { "Received #{msg.inspect} on #{Process.pid}" }
+              result = worker_proc.call(msg[:input])
+              logger.info { "Result #{result.inspect}" }
+              bunny.queue(msg[:response_queue]).publish(Marshal.dump(:result => result, :index => msg[:index]))
+            end
+          ensure
+            after_response_procs.each { |p| p.call }
           end
-        end
-      end
-    end
-  end
-
-  module ThreadStrategy
-    def start
-      @threads = []
-      count_workers.times do |c|
-        @threads << (Thread.new do
-                       begin
-                         run_consumer
-                       rescue Exception => e
-                         logger.error %(#{e}\n#{e.backtrace.join("\n")})
-                       end
-                     end)
+        rescue Qrack::ClientTimeout
+        rescue Timeout::Error
+          logger.info "Job took longer than #{timeout} seconds to complete. Aborting"
+        end while ! @shutting_down
       end
-    end
-
-    def stop(graceful = true)
-      if graceful
-        @shutting_down = true
+    rescue Exception => e # Bunny gets into a strange state when exceptions are raised, so reconnect to queue server if it happens
+      if on_exception_proc
+        on_exception_proc.call(e)
       else
-        @threads.each { |t| t.kill }
+        logger.info e.class
+        logger.error e.message
+        logger.error e.backtrace
       end
-    end
+      sleep 0.2
+    end while ! @shutting_down
+    logger.info "Done."
   end
 
   module ForkStrategy

data/lib/queue_map/version.rb

@@ -1,3 +1,3 @@
 module QueueMap
-  VERSION="0.4"
+  VERSION="0.5"
 end

data/lib/queue_map.rb

@@ -3,8 +3,9 @@ require "bunny"
 require 'timeout'
 
 module QueueMap
-  autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
   BUNNY_MUTEX = Mutex.new
+
+  autoload :Consumer, File.dirname(__FILE__) + "/queue_map/consumer"
   extend self
   attr_accessor :mode, :consumer_path
   attr_writer :consumer_base_path
@@ -41,15 +42,15 @@ module QueueMap
       response_queue = bunny.queue(response_queue_name, :durable => false, :exclusive => true, :auto_delete => true)
 
       (0..(collection.length - 1)).each do |i|
+        puts "Publishing #{collection[i].inspect}."
         q.publish(Marshal.dump(:input => collection[i], :index => i, :response_queue => response_queue_name))
       end
 
       results = {}
       begin
         Timeout.timeout(options[:timeout] || 5) do
-          collection.length.times do
-            sleep 0.05 while (next_response = response_queue.pop) == :queue_empty
-            response = Marshal.load(next_response)
+          response_queue.subscribe(:message_max => collection.length) do |msg|
+            response = Marshal.load(msg)
             results[response[:index]] = response[:result]
           end
         end
@@ -71,19 +72,26 @@ module QueueMap
   end
 
   def consumer(name)
-    consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :thread)
+    consumers[name] ||= QueueMap::Consumer.from_file(consumer_path[name], :strategy => mode || :fork)
   end
 
-  def with_bunny(&block)
-    bunny = nil
+  def new_bunny_connection
     BUNNY_MUTEX.synchronize do
       bunny = Bunny.new((@connection_info || { }).merge(:spec => '08'))
       bunny.start
+      bunny
     end
+  end
+
+  def with_bunny(&block)
+    bunny = new_bunny_connection
     begin
       yield bunny
     ensure
-      (bunny.close_connection unless bunny.status == :not_connected) rescue nil
+      BUNNY_MUTEX.synchronize do
+        bunny.stop rescue nil
+        bunny.close_connection rescue nil
+      end
     end
   end
 

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: queue_map
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 1
   prerelease: false
   segments: 
   - 0
-  - 4
-  version: "0.4"
+  - 5
+  version: "0.5"
 platform: ruby
 authors: 
 - Tim Harper
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-12 00:00:00 -06:00
+date: 2010-10-14 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency