refinery 0.9.12 → 0.9.13

data/VERSION

@@ -1 +1 @@
-0.9.12
+0.9.13

data/lib/refinery/daemon.rb

@@ -4,6 +4,7 @@ module Refinery #:nodoc:
     include Refinery::Loggable
     include Refinery::Configurable
     include Refinery::Utilities
+    include Refinery::Queueable
     
     RUNNING = 'running'
     STOPPED = 'stopped'
@@ -12,12 +13,10 @@ module Refinery #:nodoc:
     attr_reader :thread
     # The name of the daemon
     attr_reader :name
-    # The queue for incoming messages to process
-    attr_reader :waiting_queue
-    # The queue for outgoing messages once they've been processed
-    attr_reader :done_queue
-    # The queue for error messages
-    attr_reader :error_queue
+    # The settings for the daemon
+    attr_reader :settings
+    # The base queue name
+    attr_reader :queue_name
     
     # Stop the daemon
     def stop
@@ -52,56 +51,57 @@ module Refinery #:nodoc:
     # The settings hash may contain the following options:
     # * <tt>visibility</tt>: The time in seconds that the message is hidden
     # in the queue.
-    def initialize(server, name, waiting_queue, error_queue, done_queue, settings={})
+    def initialize(server, name, queue_prefix='', settings={})
       Refinery::Server.logger.debug "Starting daemon"
       
       @server = server
       @name = name
-      @waiting_queue = waiting_queue
-      @error_queue = error_queue
-      @done_queue = done_queue
+      @settings = settings
+      
+      queue_name = settings['queue'] || name
+      queue_name = "#{queue_prefix}#{queue_name}"
+      logger.debug "Using queue #{queue_name}"
+      @queue_name = queue_name
       
       @thread = Thread.new(self) do |daemon|
         logger.debug "Running daemon thread: #{name} (settings: #{settings.inspect})"
         while(running?)
-          begin
+          with_queue("#{queue_name}_waiting") do |waiting_queue|
             while (message = waiting_queue.receive(settings['visibility']))
               worker = load_worker_class(name).new(self)
               begin
                 result, run_time = worker.run(decode_message(message.body))
                 if result
-                  done_message = {
-                    'host_info' => host_info,
-                    'original' => message.body,
-                    'run_time' => run_time
-                  }
-                  logger.debug "Sending 'done' message to #{done_queue.name}"
-                  done_queue.send_message(encode_message(done_message))
+                  with_queue("#{queue_name}_done") do |done_queue|
+                    done_message = {
+                      'host_info' => host_info,
+                      'original' => message.body,
+                      'run_time' => run_time
+                    }
+                    logger.debug "Sending 'done' message to #{done_queue.name}"
+                    done_queue.send_message(encode_message(done_message))
+                  end
                 
                   logger.debug "Deleting message from queue"
                   message.delete()
                 end
               rescue Exception => e
-                error_message = {
-                  'error' => {
-                    'message' => e.message, 
-                    'class' => e.class.name
-                  }, 
-                  'host_info' => host_info,
-                  'original' => message.body
-                }
-                logger.error "Sending 'error' message to #{error_queue.name}: #{e.message}"
-                error_queue.send_message(encode_message(error_message))
+                with_queue("#{queue_name}_error") do |error_queue|
+                  error_message = {
+                    'error' => {
+                      'message' => e.message, 
+                      'class' => e.class.name
+                    }, 
+                    'host_info' => host_info,
+                    'original' => message.body
+                  }
+                  logger.error "Sending 'error' message to #{error_queue.name}: #{e.message}"
+                  error_queue.send_message(encode_message(error_message))
+                end
                 message.delete()
               end
             end
             sleep(settings['sleep'] || 5)
-          rescue Exception => e
-            logger.error "An error occurred while receiving from the waiting queue: #{e.message}"
-            # delay to try to get past the issue with the queue
-            sleep(30)
-            # assign a new queue instance
-            @waiting_queue = queue(waiting_queue.name)
           end
         end
         logger.debug "Exiting daemon thread"

data/lib/refinery/event_publisher.rb

@@ -51,9 +51,9 @@ module Refinery #:nodoc:
       raise RuntimeError, "No processor configuration found for #{key}" unless settings
       queue_name = settings['queue'] || key
       queue_name = "#{prefix}#{queue_name}"
-      logger.debug "Using queue #{queue_name}_waiting"
-      waiting_queue = queue("#{queue_name}_waiting")
-      load_publisher_class(key).new(waiting_queue).execute
+      waiting_queue_name = "#{queue_name}_waiting"
+      logger.debug "Using queue #{waiting_queue_name}"
+      load_publisher_class(key).new(waiting_queue_name).execute
     end
     
     # Run the event publisher
@@ -85,13 +85,13 @@ module Refinery #:nodoc:
         logger.info "Creating publisher for #{key}"
         queue_name = settings['queue'] || key
         queue_name = "#{prefix}#{queue_name}"
-        logger.debug "Using queue #{queue_name}_waiting"
-        waiting_queue = queue("#{queue_name}_waiting")
+        waiting_queue_name = "#{queue_name}_waiting"
+        logger.debug "Using queue #{waiting_queue_name}"
       
-        threads << Thread.new(waiting_queue, settings) do |waiting_queue, settings|
+        threads << Thread.new(waiting_queue_name, settings) do |waiting_queue_name, settings|
           while(running?)
             begin
-              load_publisher_class(key).new(waiting_queue).execute
+              load_publisher_class(key).new(waiting_queue_name).execute
             rescue Exception => e
               logger.error e
               raise e

data/lib/refinery/heartbeat.rb

@@ -9,9 +9,9 @@ module Refinery #:nodoc:
     # Initialize the heartbeat for the given server.
     def initialize(server)
       @server = server
-      @thread = Thread.new(queue('heartbeat')) do |heartbeat_queue|
+      @thread = Thread.new do
         loop do
-          begin
+          with_queue('heartbeat') do |heartbeat_queue|
             logger.debug "Send heartbeat"
             message = {
               'host_info' => host_info,
@@ -20,9 +20,6 @@ module Refinery #:nodoc:
             }
             heartbeat_queue.send_message(Base64.encode64(message.to_json))
             sleep(60)
-          rescue Exception => e
-            logger.error "Error sending heartbeat: #{e.message}"
-            sleep(30)
           end
         end
       end

data/lib/refinery/monitor.rb

@@ -49,15 +49,13 @@ module Refinery #:nodoc:
     
     def run_heartbeat_monitor
       logger.info "Starting heartbeat monitor"
-      Thread.new(queue('heartbeat')) do |heartbeat_queue|
+      Thread.new('heartbeat') do |heartbeat_queue_name|
         loop do
-          begin
+          with_queue(heartbeat_queue_name) do |heartbeat_queue|
             while (message = heartbeat_queue.receive)
               logger.debug decode_message(message.body).inspect
               message.delete()
             end
-          rescue Exception => e
-            logger.error e
           end
           sleep(5)
         end
@@ -69,9 +67,9 @@ module Refinery #:nodoc:
         queue_name = settings['queue'] || key
         done_queue_name = "#{queue_name}_done"
         logger.debug "Starting monitor for queue #{done_queue_name}"
-        Thread.new(queue(done_queue_name)) do |done_queue|
+        Thread.new(done_queue_name) do |done_queue_name|
           loop do
-            begin
+            with_queue(done_queue_name) do |done_queue|
               while (message = done_queue.receive)
                 done_message = decode_message(message.body)
                 processed = decode_message(done_message['original'])
@@ -79,10 +77,8 @@ module Refinery #:nodoc:
                 message.delete()
                 statistics.record_done(done_message)
               end
-            rescue Exception => e
-              logger.error e
+              sleep(5)
             end
-            sleep(5)
           end
         end
       end
@@ -93,9 +89,9 @@ module Refinery #:nodoc:
         queue_name = settings['queue'] || key
         error_queue_name = "#{queue_name}_error"
         logger.info "Starting error monitor for queue #{error_queue_name}"
-        Thread.new(queue(error_queue_name)) do |error_queue|
+        Thread.new(error_queue_name) do |error_queue_name|
           loop do
-            begin
+            with(error_queue_name) do |error_queue|
               while (message = error_queue.receive)
                 error_message = decode_message(message.body)
                 processed = decode_message(error_message['original'])
@@ -103,8 +99,6 @@ module Refinery #:nodoc:
                 message.delete()
                 statistics.record_error(error_message)
               end
-            rescue Exception => e
-              logger.error e
             end
             sleep(5)
           end

data/lib/refinery/publisher.rb

@@ -2,23 +2,26 @@ module Refinery #:nodoc:
   # Base class for publishers to be implemented by subclasses.
   class Publisher
     include Refinery::Loggable
+    include Refinery::Queueable
     
     # Initialize the publisher with the queue to publish messages to.
-    def initialize(waiting_queue)
-      @waiting_queue = waiting_queue
+    def initialize(waiting_queue_name)
+      @waiting_queue_name = waiting_queue_name
     end
     
     protected
-    # Get the publish queue
-    def waiting_queue
-      @waiting_queue
+    # Get the publish queue name
+    def waiting_queue_name
+      @waiting_queue_name
     end
     
     # Publish the message. The message will be converted to JSON and pushed
     # into the queue associated with the publisher.
     def publish(message)
-      logger.debug "Publisher #{self.class.name} sending message: #{message.to_json}"
-      waiting_queue.send_message(Base64.encode64(message.to_json))
+      with_queue(waiting_queue_name) do |waiting_queue|
+        logger.debug "Publisher #{self.class.name} sending message: #{message.to_json}"
+        waiting_queue.send_message(Base64.encode64(message.to_json))
+      end
     end
   end
 end

data/lib/refinery/queueable.rb

@@ -1,11 +1,23 @@
 module Refinery #:nodoc:
   # Mix this module in to classes that want to access a queue.
   module Queueable
+    include Loggable
+    include Configurable
     # Get a named queue
     def queue(name)
       queue_provider.queue(name)
     end
     
+    def with_queue(name, &block)
+      begin
+        yield queue(name)
+      rescue Exception => e
+        logger.error "Queue error: #{e.message}"
+        sleep(5)
+        retry
+      end
+    end
+    
     protected
     # Get the queue provider. Defaults to RightAws::SqsGen2 running
     # in multi-thread mode.

data/lib/refinery/server.rb

@@ -60,19 +60,11 @@ module Refinery #:nodoc:
     
     private
     def execute_daemons
-      prefix = config['prefix'] || ''
+      queue_prefix = config['prefix'] || ''
       config['processors'].each do |key, settings|
         logger.debug "Creating daemons for #{key}"
-        
-        queue_name = settings['queue'] || key
-        queue_name = "#{prefix}#{queue_name}"
-        logger.debug "Using queue #{queue_name}"
-        waiting_queue = queue("#{queue_name}_waiting")
-        error_queue = queue("#{queue_name}_error")
-        done_queue = queue("#{queue_name}_done")
-        
         1.upto(settings['workers']['initial']) do
-          daemons << Refinery::Daemon.new(self, key, waiting_queue, error_queue, done_queue, settings)
+          daemons << Refinery::Daemon.new(self, key, queue_prefix, settings)
         end
         
         logger.debug "Running #{daemons.length} daemons"

data/refinery.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{refinery}
-  s.version = "0.9.12"
+  s.version = "0.9.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Anthony Eden"]
-  s.date = %q{2009-07-08}
+  s.date = %q{2009-07-09}
   s.description = %q{Process data in a distributed fashion.}
   s.email = %q{anthonyeden@gmail.com}
   s.executables = ["epub", "monitor", "pubnow", "refinery"]

data/test/unit/daemon_test.rb

@@ -6,26 +6,40 @@ class DaemonTest < Test::Unit::TestCase
       @waiting_queue = stub('Queue(waiting)')
       @error_queue = stub('Queue(error)')
       @done_queue = stub('Queue(done)')
+      
+      Refinery::Daemon.any_instance.stubs(:queue).with(
+      'sample_waiting').returns(@waiting_queue)
+      Refinery::Daemon.any_instance.stubs(:queue).with(
+      'sample_error').returns(@error_queue)
+      Refinery::Daemon.any_instance.stubs(:queue).with(
+      'sample_done').returns(@done_queue)
     end
     should "be startable" do
       @waiting_queue.stubs(:receive)
       assert_nothing_raised do
-        daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+        daemon = Refinery::Daemon.new(@server, 'sample')
       end
     end
     should "have logging" do
       @waiting_queue.stubs(:receive)
-      daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+      daemon = Refinery::Daemon.new(@server, 'sample')
       assert_not_nil daemon.logger
     end
     should "allow visibility setting" do
       @waiting_queue.expects(:receive).with(600)
-      daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue, {'visibility' => 600})
+      daemon = Refinery::Daemon.new(@server, 'sample', '', {'visibility' => 600})
+    end
+    should "have a queue name" do
+      @waiting_queue.stubs(:receive)
+      Refinery::Daemon.any_instance.stubs(:queue).with(
+      'prefix_sample_waiting').returns(@waiting_queue)
+      daemon = Refinery::Daemon.new(@server, 'sample', 'prefix_')
+      assert_equal 'prefix_sample', daemon.queue_name
     end
     context "that is started" do
       setup do
         @waiting_queue.stubs(:receive)
-        @daemon = Refinery::Daemon.new(@server, 'sample', @waiting_queue, @error_queue, @done_queue)
+        @daemon = Refinery::Daemon.new(@server, 'sample')
       end
       should "have a state of running" do
         assert @daemon.running?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: refinery
 version: !ruby/object:Gem::Version 
-  version: 0.9.12
+  version: 0.9.13
 platform: ruby
 authors: 
 - Anthony Eden
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-08 00:00:00 -04:00
+date: 2009-07-09 00:00:00 -04:00
 default_executable: 
 dependencies: []