qwirk 0.0.1 → 0.1.0

data/lib/qwirk/batch/file_worker.rb

@@ -1,7 +1,7 @@
 module Qwirk
   module Batch
 
-    # Batch worker which reads records from files and queues them up for a separate worker (Qwirk::QueueAdapter::JMS::RequestWorker) to process.
+    # Batch worker which reads records from files and queues them up for a separate worker (Qwirk::Adapter::JMS::ReplyWorker) to process.
     # For instance, a worker of this type might look as follows:
     #   class MyBatchWorker
     #     include Qwirk::Batch::FileWorker
@@ -204,18 +204,18 @@ module Qwirk
       #######
 
       def reply_event_loop
-        @reply_session = Qwirk::QueueAdapter::JMS::Connection.create_session
+        @reply_session = Qwirk::Adapter::JMS::Connection.create_session
         @consumer = @reply_session.consumer(:queue_name => @queue_name)
         @reply_session.start
 
         while !@stopped && message = @consumer.receive
           @message_mutex.synchronize do
-            obj = Qwirk::QueueAdapter::JMS.parse_response(message)
+            obj = Qwirk::Adapter::JMS.parse_response(message)
             process_response(obj)
             message.acknowledge
           end
           # TODO: @time_track now in WorkerConfig
-          #Qwirk.logger.info {"#{self}::on_message (#{('%.1f' % (@time_track.last_time*1000.0))}ms)"} if Qwirk::QueueAdapter::JMS::Connection.log_times?
+          #Qwirk.logger.info {"#{self}::on_message (#{('%.1f' % (@time_track.last_time*1000.0))}ms)"} if Qwirk::Adapter::JMS::Connection.log_times?
         end
         @status = 'Exited'
         Qwirk.logger.info "#{self}: Exiting"

data/lib/qwirk/manager.rb

@@ -25,9 +25,9 @@ module Qwirk
     #     production, a set of workers could be defined under production or specific workers for each host name.
     #   persist_file - WorkerConfig attributes that are modified externally (via Rumx interface) will be stored in this file.  Without this
     #     option, external config changes that are made will be lost when the Manager is restarted.
-    def initialize(queue_adapter, options={})
+    def initialize(adapter_factory, options={})
+      @adapter_factory    = adapter_factory
       options           = @@default_options.merge(options)
-      #puts "creating qwirk manager with #{options.inspect}"
       @stopped          = false
       @name             = options[:name] || Qwirk::DEFAULT_NAME
       @poll_time        = 3.0
@@ -38,13 +38,13 @@ module Qwirk
       @persist_options  = (@persist_file && File.exist?(@persist_file)) ? YAML.load_file(@persist_file) : {}
 
       BaseWorker.worker_classes.each do |worker_class|
-        worker_config_class = worker_class.config_class
-        worker_class.each_config do |config_name, default_options|
-          # Least priority is config options defined in the Worker class, then the workers.yml file, highest priority is persist_file (ad-hoc changes made manually)
+        worker_class.each_config(adapter_factory.worker_config_class) do |config_name, extended_worker_config_class, default_options|
+          # Least priority is config default_options defined in the Worker class, then the workers.yml file,
+          # highest priority is persist_file (ad-hoc changes made manually)
           options = {}
           options = options.merge(@worker_options[config_name]) if @worker_options[config_name]
           options = options.merge(@persist_options[config_name]) if @persist_options[config_name]
-          worker_config = worker_config_class.new(queue_adapter, config_name, self, worker_class, default_options, options)
+          worker_config = extended_worker_config_class.new(adapter_factory, config_name, self, worker_class, default_options, options)
           bean_add_child(config_name, worker_config)
           @worker_configs << worker_config
         end
@@ -54,6 +54,8 @@ module Qwirk
       stop_on_signal if options[:stop_on_signal]
     end
 
+    # Create a timer_thread to make periodic calls to the worker_configs in order to do such things as expand/contract
+    # workers, etc.
     def start_timer_thread
       # TODO: Optionize hard-coded values
       @timer_thread = Thread.new do
@@ -86,12 +88,13 @@ module Qwirk
       end
     end
 
+    # Store off any options that are no longer set to default
     def save_persist_state
       return unless @persist_file
       new_persist_options = {}
       BaseWorker.worker_classes.each do |worker_class|
-        worker_class.each_config do |config_name, options|
-          static_options = options.merge(@worker_options[config_name] || {})
+        worker_class.each_config(@adapter_factory.worker_config_class) do |config_name, ignored_extended_worker_config_class, default_options|
+          static_options = default_options.merge(@worker_options[config_name] || {})
           worker_config = self[config_name]
           hash = {}
           # Only store off the config values that are specifically different from default values or values set in the workers.yml file

data/lib/qwirk/marshal_strategy/none.rb

@@ -1,6 +1,6 @@
 module Qwirk
   module MarshalStrategy
-    # Should only be used with InMem strategy
+    # Should only be used with InMemory strategy
     module None
       extend self
 

data/lib/qwirk/publish_handle.rb

@@ -3,9 +3,14 @@ require 'timeout'
 module Qwirk
   class PublishHandle
     def initialize(publisher, adapter_info, start)
-      @producer    = publisher
+      @producer     = publisher
       @adapter_info = adapter_info
       @start        = start
+      @timeout      = false
+    end
+
+    def timeout?
+      @timeout
     end
 
     # Waits the given timeout for a response message on the queue.
@@ -54,13 +59,19 @@ module Qwirk
       # Creates a block for reading the responses for a given message_id (adapter_info).  The block will be passed an object
       # that responds to timeout_read(timeout) with a [original_message_id, response_message, worker_name] tri or nil if no message is read.
       # This is used in the RPC mechanism where a publish might wait for 1 or more workers to respond.
-      @producer.adapter.with_response(@adapter_info) do |consumer|
+      @producer.impl.with_response(@adapter_info) do |consumer|
         if block_given?
           return read_multiple_response(consumer, timeout, &block)
         else
-          response = read_single_response(consumer, timeout)
-          raise response if response.kind_of?(Qwirk::RemoteException)
-          return response
+          tri = read_single_response(consumer, timeout)
+          if tri
+            response = tri[1]
+            raise response if response.kind_of?(Qwirk::RemoteException)
+            return response
+          else
+            @timeout = !tri
+            return nil
+          end
         end
       end
     end
@@ -71,8 +82,7 @@ module Qwirk
 
     def read_single_response(consumer, timeout)
       leftover_timeout = @start + timeout - Time.now
-      message_id, response, @worker_name = consumer.timeout_read(leftover_timeout)
-      return response
+      return consumer.timeout_read(leftover_timeout)
     end
 
     def read_multiple_response(consumer, timeout, &block)
@@ -80,15 +90,16 @@ module Qwirk
       yield worker_response
 
       until worker_response.done? do
-        response = read_single_response(consumer, timeout)
-        if !response
+        tri = read_single_response(consumer, timeout)
+        if !tri
           worker_response.make_timeout_calls
           return
         end
+        ignored_message_id, response, worker_name = tri
         if response.kind_of?(Qwirk::RemoteException)
-          worker_response.make_exception_call(@worker_name, response)
+          worker_response.make_exception_call(worker_name, response)
         else
-          worker_response.make_message_call(@worker_name, response)
+          worker_response.make_message_call(worker_name, response)
         end
       end
     end

data/lib/qwirk/publisher.rb

@@ -4,7 +4,7 @@ module Qwirk
     include Rumx::Bean
 
     #attr_reader :producer_options, :persistent, :reply_queue
-    attr_reader :response_options, :adapter, :marshaler
+    attr_reader :response_options, :impl, :marshaler
 
     bean_attr_reader :tasks, :hash, 'Hash of the latest tasks', :hash_type => :bean
 
@@ -19,7 +19,7 @@ module Qwirk
     #     :response              => if true or a hash of response options, a temporary reply queue will be setup for handling responses
     #       :time_to_live        => expiration time in ms for the response message(s) (JMS))
     #       :persistent          => true or false for the response message(s), set to false if you don't want timed out messages ending up in the DLQ (defaults to true unless time_to_live is set)
-    def initialize(queue_adapter, options)
+    def initialize(adapter_factory, options)
       options = options.dup
       @queue_name = options.delete(:queue_name)
       @topic_name = options.delete(:topic_name)
@@ -29,17 +29,17 @@ module Qwirk
       # response_options should only be a hash or the values true or false
       @response_options = {} if @response_options && !@response_options.kind_of?(Hash)
 
-      @tasks            = {}
-      @adapter          = queue_adapter.create_adapter_publisher(@queue_name, @topic_name, options, @response_options)
-      marshal_sym       = options[:marshal] || :ruby
-      @marshaler        = Qwirk::MarshalStrategy.find(marshal_sym)
+      @tasks      = {}
+      @impl       = adapter_factory.create_publisher_impl(@queue_name, @topic_name, options, @response_options)
+      marshal_sym = options[:marshal] || :ruby
+      @marshaler  = Qwirk::MarshalStrategy.find(marshal_sym)
     end
 
     # Publish the given object to the address.
     def publish(object, props={})
       start = Time.now
       marshaled_object = @marshaler.marshal(object)
-      adapter_info = @adapter.publish(marshaled_object, @marshaler, nil, props)
+      adapter_info = @impl.publish(marshaled_object, @marshaler, nil, props)
       return PublishHandle.new(self, adapter_info, start)
     end
 
@@ -49,7 +49,7 @@ module Qwirk
     # last message read.  It should also respond to stop which will interrupt any receive calls causing it to return nil.
     def create_producer_consumer_pair(task)
       @tasks[task.task_id] = task
-      @adapter.create_producer_consumer_pair(task.task_id, @marshaler)
+      @impl.create_producer_consumer_pair(task.task_id, @marshaler)
     end
 
     def to_s
@@ -61,7 +61,7 @@ module Qwirk
     end
 
     def create_fail_queue_consumer(fail_queue_name=nil)
-      fail_queue_name ||= default_fail_queue_name
+      fail_queue_name || default_fail_queue_name
     end
   end
 end

data/lib/qwirk/{request_worker.rb → reply_worker.rb}

@@ -1,7 +1,7 @@
 module Qwirk
 
   # Base Worker Class for any class that will be processing requests from queues and replying
-  module RequestWorker
+  module ReplyWorker
     include Worker
 
     module ClassMethods
@@ -27,7 +27,7 @@ module Qwirk
       rescue Exception => e
         on_exception(e)
       else
-        adapter.send_response(message, config.marshaler.marshal(response))
+        impl.send_response(message, config.marshaler.marshal(response))
       end
       post_request(object)
     rescue Exception => e
@@ -49,7 +49,7 @@ module Qwirk
 
     def on_exception(e)
       begin
-        adapter.send_exception(message, e)
+        impl.send_exception(message, e)
       rescue Exception => e
         Qwirk.logger.error("Exception in exception reply: #{e.message}")
         log_backtrace(e)

data/lib/qwirk/worker.rb

@@ -4,7 +4,7 @@ module Qwirk
   # By default, it will consume messages from a queue with the class name minus the Worker postfix.
   # For example, the queue call is unnecessary as it will default to a value of 'Foo' anyways:
   #  class FooWorker
-  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    include Qwirk::Worker
   #    queue 'Foo'
   #    def perform(obj)
   #      # Perform work on obj
@@ -15,7 +15,7 @@ module Qwirk
   # each thread for a given worker will act as a separate subscriber.
   # (For ActiveMQ - see http://activemq.apache.org/virtual-destinations.html):
   #  class FooWorker
-  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    include Qwirk::Worker
   #    topic 'Zulu'
   #    def perform(obj)
   #      # Perform work on obj
@@ -25,7 +25,7 @@ module Qwirk
   # TODO (maybe):
   # Filters can also be specified within the class:
   #  class FooWorker
-  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    include Qwirk::Worker
   #    filter 'age > 30'
   #    def perform(obj)
   #      # Perform work on obj
@@ -37,7 +37,7 @@ module Qwirk
     include Qwirk::BaseWorker
 
     attr_accessor :message
-    attr_reader   :status, :adapter, :start_worker_time, :start_read_time, :start_processing_time
+    attr_reader   :status, :impl, :start_worker_time, :start_read_time, :start_processing_time
 
     module ClassMethods
       def queue(name, opts={})
@@ -71,13 +71,12 @@ module Qwirk
       end
 
       # Defines the default value of the fail_queue_target.  For extenders of this class, the default will be true
-      # but extenders can change this (RequestWorker returns exceptions to the caller so it defaults to false).
+      # but extenders can change this (ReplyWorker returns exceptions to the caller so it defaults to false).
       def default_fail_queue_target
         true
       end
 
       def queue_name(default_name)
-        puts "getting queue_name queue=#{@queue_name} topic=#{@topic_name} default=#{default_name}"
         return @queue_name if @queue_name
         return nil if @topic_name
         return default_name
@@ -113,13 +112,16 @@ module Qwirk
       base.extend(ClassMethods)
     end
 
-    def start(index, worker_config)
-      @status               = 'Started'
-      @stopped              = false
-      @processing_mutex     = Mutex.new
-      self.index            = index
-      self.config           = worker_config
-      @adapter              = worker_config.adapter.create_worker
+    def init(index, worker_config)
+      @status           = 'Started'
+      @stopped          = false
+      @processing_mutex = Mutex.new
+      self.index        = index
+      self.config       = worker_config
+      @impl             = worker_config.create_worker
+    end
+
+    def start
       self.thread = Thread.new do
         java.lang.Thread.current_thread.name = "Qwirk worker: #{self}" if RUBY_PLATFORM == 'jruby'
         #Qwirk.logger.debug "#{worker}: Started thread with priority #{Thread.current.priority}"
@@ -131,17 +133,17 @@ module Qwirk
     # clean up any resources.  We don't want to clobber resources while a message is being processed so processing_mutex will surround
     # message processessing and worker closing.
     # From a JMS perspective, stop all workers (close consumer and session), stop the config.
-    # From an InMem perspective, we don't want the workers stopping until all messages in the queue have been processed.
+    # From an InMemory perspective, we don't want the workers stopping until all messages in the queue have been processed.
     # Therefore we want to stop the
     def stop
-      puts "#{self}: In base worker stop"
+      Qwirk.logger.debug "#{self}: In base worker stop"
       @status  = 'Stopping'
       @stopped = true
       @processing_mutex.synchronize do
-        # This should interrupt @adapter.receive_message above and cause it to return nil
-        @adapter.stop
+        # This should interrupt @impl.receive_message above and cause it to return nil
+        @impl.stop
       end
-      puts "#{self}: base worker stop complete"
+      Qwirk.logger.debug "#{self}: base worker stop complete"
     end
 
     def perform(object)
@@ -157,25 +159,21 @@ module Qwirk
       Qwirk.logger.error "\t#{e.backtrace.join("\n\t")}"
     end
 
-    #########
-    protected
-    #########
-
     # Start the event loop for handling messages off the queue
     def event_loop
       Qwirk.logger.debug "#{self}: Starting receive loop"
       @start_worker_time = Time.now
-      while !@stopped && !config.adapter.stopped
-        puts "#{self}: Waiting for read"
+      while !@stopped && !config.stopped
+        Qwirk.logger.debug "#{self}: Waiting for read"
         @start_read_time = Time.now
-        msg = @adapter.receive_message
+        msg = @impl.receive_message
         if msg
           @start_processing_time = Time.now
           Qwirk.logger.debug {"#{self}: Done waiting for read in #{@start_processing_time - @start_read_time} seconds"}
           delta = config.timer.measure do
             @processing_mutex.synchronize do
               on_message(msg)
-              @adapter.acknowledge_message(msg)
+              @impl.acknowledge_message(msg)
             end
           end
           Qwirk.logger.info {"#{self}::on_message (#{'%.1f' % delta}ms)"} if self.config.log_times
@@ -189,7 +187,7 @@ module Qwirk
     ensure
       @status = 'Stopped'
       # TODO: necessary?
-      @adapter.stop
+      @impl.stop
       Qwirk.logger.flush if Qwirk.logger.respond_to?(:flush)
       config.worker_stopped(self)
     end
@@ -197,7 +195,7 @@ module Qwirk
     def on_message(message)
       # TBD - Is it necessary to provide underlying message to worker?  Should we generically provide access to message attributes?  Do filters somehow fit in here?
       @message = message
-      object = @adapter.message_to_object(message)
+      object = @impl.message_to_object(message)
       Qwirk.logger.debug {"#{self}: Received Object: #{object}"}
       perform(object)
     rescue Exception => e
@@ -209,7 +207,7 @@ module Qwirk
     def on_exception(e)
       Qwirk.logger.error "#{self}: Messaging Exception: #{e.message}"
       log_backtrace(e)
-      @adapter.handle_failure(message, @fail_queue_name) if @fail_queue_name
+      @impl.handle_failure(message, e, @fail_queue_name) if @fail_queue_name
     rescue Exception => e
       Qwirk.logger.error "#{self}: Exception in exception reply: #{e.message}"
       log_backtrace(e)

data/test/jms_fail_test.rb

@@ -7,7 +7,7 @@ require 'erb'
 # NOTE: This test requires a running ActiveMQ server
 
 class ExceptionWorker
-  include Qwirk::QueueAdapter::JMS::Worker
+  include Qwirk::Adapter::JMS::Worker
   def perform(obj)
     puts "#{name} received #{obj} but raising exception"
     raise 'foobar'
@@ -17,8 +17,8 @@ class ExceptionWorker
   end
 end
 
-class ExceptionRequestWorker
-  include Qwirk::QueueAdapter::JMS::RequestWorker
+class ExceptionReplyWorker
+  include Qwirk::Adapter::JMS::ReplyWorker
 
   def request(obj)
     puts "#{name} received #{obj} but raising exception"
@@ -31,7 +31,7 @@ end
 
 # This will read from the queue that ExceptionWorker fails to
 class ExceptionFailWorker
-  include Qwirk::QueueAdapter::JMS::Worker
+  include Qwirk::Adapter::JMS::Worker
 
   @@my_hash = {}
 
@@ -49,7 +49,7 @@ class JMSFailTest < Test::Unit::TestCase
 
   def assert_fail_queue(queue_name, fail_queue_name, value, is_fail_queue_expected)
     # Publish to Exception that will throw exception which will put on ExceptionFail queue
-    publisher = Qwirk::QueueAdapter::JMS::Publisher.new(:queue_name => queue_name, :marshal => :string)
+    publisher = Qwirk::Adapter::JMS::Publisher.new(:queue_name => queue_name, :marshal => :string)
     puts "Publishing #{value} to #{queue_name}"
     publisher.publish(value)
     sleep 1
@@ -60,7 +60,7 @@ class JMSFailTest < Test::Unit::TestCase
   context 'jms' do
     setup do
       config = YAML.load(ERB.new(File.read(File.join(File.dirname(__FILE__), 'jms.yml'))).result(binding))
-      Qwirk::QueueAdapter::JMS::Connection.init(config)
+      Qwirk::Adapter::JMS::Connection.init(config)
     end
 
     teardown do
@@ -97,28 +97,28 @@ class JMSFailTest < Test::Unit::TestCase
 
         # Should NOT receive message on the fail worker
         name = 'ExceptionRequest'
-        @manager.add(ExceptionRequestWorker, 1)
+        @manager.add(ExceptionReplyWorker, 1)
         @manager.add(ExceptionFailWorker, 1, :name => "#{name}Fail")
 
         # Should NOT receive message on the fail worker when using specified names
         name = 'ExceptionRequestNameSpecified'
-        @manager.add(ExceptionRequestWorker, 1, :name => name)
+        @manager.add(ExceptionReplyWorker, 1, :name => name)
         @manager.add(ExceptionFailWorker, 1, :name => "#{name}Fail")
 
         # Should receive message on the fail worker when using specified names and fail_queue set true
         name = 'ExceptionRequestFailQueueTrue'
-        @manager.add(ExceptionRequestWorker, 1, :name => name, :fail_queue => true)
+        @manager.add(ExceptionReplyWorker, 1, :name => name, :fail_queue => true)
         @manager.add(ExceptionFailWorker, 1, :name => "#{name}Fail")
 
         # Should NOT receive message on the fail worker when using specified names and fail_queue set false
         name = 'ExceptionRequestFailQueueFalse'
-        @manager.add(ExceptionRequestWorker, 1, :name => name, :fail_queue => false)
+        @manager.add(ExceptionReplyWorker, 1, :name => name, :fail_queue => false)
         @manager.add(ExceptionFailWorker, 1, :name => "#{name}Fail")
 
         # Should NOT receive message on the fail worker when using specified names and fail_queue set false
         name = 'ExceptionRequestFailQueueSpecified'
         fail_queue = 'MyRequestFailQueue'
-        @manager.add(ExceptionRequestWorker, 1, :name => name, :fail_queue => fail_queue)
+        @manager.add(ExceptionReplyWorker, 1, :name => name, :fail_queue => fail_queue)
         @manager.add(ExceptionFailWorker, 1, :name => fail_queue)
 
         sleep 1