qwirk 0.0.1 → 0.1.0

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/factory.rb

@@ -1,6 +1,6 @@
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
       module Factory
         class << self
           def init

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/publisher.rb

@@ -1,9 +1,9 @@
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
       class Publisher
 
-        def initialize(queue_adapter, queue_name, topic_name, options, response_options)
+        def initialize(adapter_factory, queue_name, topic_name, options, response_options)
           @queue_name, @topic_name, @options, @response_options = queue_name, topic_name, options, response_options
           @queue = Factory.get_publisher_queue(queue_name, topic_name)
         end
@@ -42,7 +42,7 @@ module Qwirk
 
         def create_fail_producer_consumer_pair(task_id, marshaler)
           consumer_queue          = Queue.new("#{@queue}Fail:#{task_id}")
-          # TODO: Unlimitied queue or some form of exception on maximum
+          # TODO: Unlimited queue or some form of exception on maximum
           consumer_queue.max_size = -1
           producer = MyTaskProducer.new(@queue, consumer_queue, marshaler, {})
           consumer  = MyTaskConsumer.new(@queue, consumer_queue)

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/queue.rb

@@ -1,6 +1,6 @@
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
 
       class Queue
         # TODO: Look into reimplementing using a Ruby Queue which is probably better performant
@@ -24,6 +24,7 @@ module Qwirk
         end
 
         def stop
+          return if @stopped
           @stopped = true
           @outstanding_hash_mutex.synchronize do
             @write_condition.broadcast

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/reply_queue.rb

@@ -1,6 +1,6 @@
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
 
       class ReplyQueue
         def initialize(name)
@@ -47,6 +47,7 @@ module Qwirk
               @read_condition.wait(@outstanding_hash_mutex)
             end
           rescue Timeout::Error => e
+            return nil
           end
         end
 

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/topic.rb

@@ -1,6 +1,6 @@
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
 
       class Topic
         def initialize(name)

data/lib/qwirk/{queue_adapter/in_mem → adapter/in_memory}/worker.rb

@@ -1,16 +1,14 @@
 # Handle Messaging and Queuing using JMS
 module Qwirk
-  module QueueAdapter
-    module InMem
+  module Adapter
+    module InMemory
       class Worker
         attr_reader :stopped
 
-        def initialize(name, marshaler, queue, queue_name, topic_name)
+        def initialize(name, marshaler, queue)
           @name       = name
           @marshaler  = marshaler
           @queue      = queue
-          @queue_name = queue_name
-          @topic_name = topic_name
         end
 
         def receive_message
@@ -36,7 +34,7 @@ module Qwirk
           return msg
         end
 
-        def handle_failure(message, fail_queue_name)
+        def handle_failure(message, exception, fail_queue_name)
           # TODO: Mode for persisting to flat file?
           Qwirk.logger.warn("Dropping message that failed: #{message}")
         end
@@ -48,11 +46,11 @@ module Qwirk
           @queue.interrupt_read
         end
 
-        ## End of required override methods for worker adapter
+        ## End of required override methods for worker impl
         private
 
         def do_send_response(original_message, object)
-          puts "Returning #{object} to queue #{@reply_queue}"
+          Qwirk.logger.debug { "Returning #{object} to queue #{@reply_queue}" }
           return unless @reply_queue
           @reply_queue.write([original_message.object_id, object, @name])
           return true

data/lib/qwirk/adapter/in_memory/worker_config.rb

@@ -0,0 +1,50 @@
+# Handle Messaging and Queuing using JMS
+module Qwirk
+  module Adapter
+    module InMemory
+
+      class WorkerConfig < Qwirk::Adapter::Base::ExpandingWorkerConfig
+
+        bean_reader   :queue_size,     :integer, 'Current count of messages in the queue'
+        bean_accessor :queue_max_size, :integer, 'Max messages allowed in the queue', :config_item => true
+
+        def self.default_marshal_sym
+          :none
+        end
+
+        def self.initial_default_config
+          super.merge(:queue_max_size => 100)
+        end
+
+        def init
+          super
+          @queue = Factory.get_worker_queue(self.name, self.queue_name, self.topic_name, @queue_max_size)
+        end
+
+        def create_worker
+          Worker.new(self.name, self.marshaler, @queue)
+        end
+
+        def stop
+          Qwirk.logger.debug { "Stopping #{self}" }
+          @queue.stop
+          super
+        end
+
+        def queue_size
+          return 0 unless @queue
+          @queue.size
+        end
+
+        def queue_max_size
+          @queue_max_size
+        end
+
+        def queue_max_size=(max_size)
+          @queue_max_size = max_size
+          @queue.max_size = max_size if @queue
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter/inline.rb

@@ -0,0 +1,9 @@
+require 'qwirk/adapter/inline/publisher'
+require 'qwirk/adapter/inline/worker_config'
+require 'qwirk/adapter/inline/worker'
+
+::Qwirk.register_adapter(
+    :inline,
+    ::Qwirk::Adapter::Inline::Publisher,
+    ::Qwirk::Adapter::Inline::WorkerConfig
+)

data/lib/qwirk/adapter/inline/publisher.rb

@@ -0,0 +1,86 @@
+module Qwirk
+  module Adapter
+    module Inline
+      class Publisher
+
+        class MyResponseHandle
+          def initialize
+            @responses = []
+          end
+
+          def add(original_message_id, response_message, worker_name)
+            @responses << [original_message_id, response_message, worker_name]
+          end
+
+          # We're inline so we either have a response or not (not is interpreted as a timeout)
+          def timeout_read(timeout)
+            @responses.pop
+          end
+        end
+
+        def initialize(adapter_factory, queue_name, topic_name, options, response_options)
+          @adapter_factory, @queue_name, @topic_name, @options, @response_options = adapter_factory, queue_name, topic_name, options, response_options
+        end
+
+        def default_marshal_sym
+          :none
+        end
+
+        # Publish the given object and return the reply_queue as the adapter_info.
+        def publish(marshaled_object, marshaler, task_id, props)
+          response_handle = @response_options ? MyResponseHandle.new : nil
+          # Since we're inline, we'll just unmarshal the object so there is less info to carry around
+          object = marshaler.unmarshal(marshaled_object)
+          if manager = @adapter_factory.manager
+            @message_handled = false
+            manager.worker_configs.each do |worker_config|
+              if worker_config.active
+                if @queue_name && @queue_name == worker_config.queue_name
+                  run_worker(worker_config, object, response_handle)
+                  @message_handled = true
+                  break
+                elsif @topic_name && @topic_name == worker_config.topic_name
+                  run_worker(worker_config, object, response_handle)
+                  @message_handled = true
+                end
+              end
+            end
+            if !@message_handled && !@no_message_handled_warning
+              Qwirk.logger.warn("Publish message #{object.inspect} being dropped as no Qwirk worker has been configured to handle it")
+              @no_message_handled_warning = true
+            end
+          elsif !@no_manager_warning
+            Qwirk.logger.warn("Publish message #{object.inspect} being dropped as no Qwirk manager has been configured for #{@adapter_factory.key}")
+            @no_manager_warning = true
+          end
+          return response_handle
+        end
+
+        # See Qwirk::PublishHandle#read_response for the requirements for this method.
+        def with_response(response_handle, &block)
+          raise "Could not find response_handle" unless response_handle
+          yield response_handle
+        end
+
+        # See Qwirk::Publisher#create_producer_consumer_pair for the requirements for this method
+        def create_producer_consumer_pair(task_id, marshaler)
+          # TBD
+        end
+
+        def create_fail_producer_consumer_pair(task_id, marshaler)
+          # TBD
+        end
+
+        #######
+        private
+        #######
+        def run_worker(worker_config, object, response_handle)
+          worker = worker_config.worker_class.new
+          worker.init(0, worker_config)
+          worker.impl.response_handle = response_handle
+          worker.on_message(object)
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter/inline/worker.rb

@@ -0,0 +1,55 @@
+# Handle Messaging and Queuing using JMS
+module Qwirk
+  module Adapter
+    module Inline
+      class Worker
+
+        attr_accessor :response_handle
+
+        def initialize(name, marshaler)
+          @name       = name
+          @marshaler  = marshaler
+        end
+
+        # We never call worker.start so this method is unnecessary
+        def receive_message
+        end
+
+        # We never call worker.start so this method is unnecessary
+        def acknowledge_message(message)
+        end
+
+        def send_response(original_message, marshaled_object)
+          # We unmarshal so our workers get consistent messages regardless of the adapter
+          do_send_response(original_message, @marshaler.unmarshal(marshaled_object))
+        end
+
+        def send_exception(original_message, e)
+          do_send_response(original_message, Qwirk::RemoteException.new(e))
+        end
+
+        def message_to_object(msg)
+          # The publisher has already unmarshaled the object to save hassle here.
+          return msg
+        end
+
+        def handle_failure(message, exception, fail_queue_name)
+          Qwirk.logger.warn("Dropping message that failed: #{message}")
+        end
+
+        def stop
+        end
+
+        ## End of required override methods for worker impl
+        private
+
+        def do_send_response(original_message, object)
+          Qwirk.logger.debug { "Returning #{object} to queue #{@response_handle}" }
+          return false unless @response_handle
+          @response_handle.add(original_message.object_id, object, @name)
+          return true
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter/inline/worker_config.rb

@@ -0,0 +1,30 @@
+# Handle Messaging and Queuing using JMS
+module Qwirk
+  module Adapter
+    module Inline
+
+      class WorkerConfig < Qwirk::Adapter::Base::WorkerConfig
+
+        bean_attr_accessor :active, :boolean, 'Whether this worker is active or not', :config_item => true
+
+        def self.default_marshal_sym
+          :none
+        end
+
+        # Define the default config values for the attributes all workers will share.  These will be sent as options to the constructor
+        def self.initial_default_config
+          super.merge(:active => false)
+        end
+
+        # Hack - Steal attribute from expanding_worker_config so test config can share development config
+        def min_count=(min_count)
+          @active = (min_count > 0)
+        end
+
+        def create_worker
+          Worker.new(self.name, self.marshaler)
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter_factory.rb

@@ -0,0 +1,48 @@
+module Qwirk
+
+  # Defines the queuing adapter.  Currently, only JMS and InMemory.
+  class AdapterFactory
+    include Rumx::Bean
+
+    attr_reader :key, :config, :log_times, :adapter_info, :worker_config_class, :manager
+
+    @@adapter_hash = {}
+
+    # Register an adapter by passing in a publisher_class, a worker_config_class and optionally a code block.
+    # The code block will expect a config object as it's argument and will return connection information
+    # or any other client_information required for this adapter.
+    def self.register(key, publisher_class, worker_config_class, &block)
+      @@adapter_hash[key] = [publisher_class, worker_config_class, block]
+    end
+
+    def initialize(key, config)
+      @key       = key
+      @config    = config.dup
+      @log_times = config.delete(:log_times)
+
+      key = config.delete(:adapter)
+      raise "No adapter definition" unless key
+      key = key.to_sym
+      @publisher_class, @worker_config_class, block = @@adapter_hash[key]
+      raise "No adapter matching #{key}" unless @publisher_class
+      @adapter_info = block.call(config) if block
+    end
+
+    def create_publisher(options={})
+      @publisher_parent ||= Rumx::Beans::Folder.new
+      publisher = Publisher.new(self, @config.merge(options))
+      @publisher_parent.bean_add_child(publisher.to_s, publisher)
+      return publisher
+    end
+
+    def create_manager(options={})
+      @manager = Manager.new(self, @config.merge(options))
+      bean_add_child(:Workers, @manager)
+      return @manager
+    end
+
+    def create_publisher_impl(queue_name, topic_name, options, response_options)
+      @publisher_class.new(self, queue_name, topic_name, options, response_options)
+    end
+  end
+end

data/lib/qwirk/base_worker.rb

@@ -1,7 +1,7 @@
 require 'rumx'
 
 module Qwirk
-  # TODO: Is this necessary anymore or just put in worker.rb?  Decide when flat file queue_adapter is implemented.
+  # TODO: Is this necessary anymore or just put in worker.rb?  Decide when flat file adapter is implemented.
   module BaseWorker
 
     attr_accessor :index, :thread, :config
@@ -12,52 +12,46 @@ module Qwirk
         name.sub(/::/, '_')
       end
 
-      # Dynamic class create form WorkerConfig and extended through config_accessor, etc calls that will be defined in the worker
-      def config_class
-        @config_class ||= Class.new(WorkerConfig)
-      end
-
-      # Default values for all the config attributes
-      def default_config
-        @default_config ||= WorkerConfig.initial_default_config
+      def bean_attrs
+        @bean_attrs ||= []
       end
 
       #config_accessor :sleep_time, :float, 'Number of seconds to sleep between messages', 5
       def config_accessor(name, type, description, default_value=nil)
-        make_bean_attr(:bean_attr_accessor, name, type, description, default_value)
+        self.bean_attrs << [:bean_attr_accessor, name, type, description, default_value]
       end
 
       def config_reader(name, type, description, default_value=nil)
-        make_bean_attr(:bean_attr_reader, name, type, description, default_value)
+        self.bean_attrs << [:bean_attr_reader, name, type, description, default_value]
       end
 
       def config_writer(name, type, description, default_value=nil)
-        make_bean_attr(:bean_attr_writer, name, type, description, default_value)
+        self.bean_attrs << [:bean_attr_writer, name, type, description, default_value]
       end
 
       def define_configs(configs)
         @configs = configs
       end
 
-      def each_config(&block)
+      # For each configuration of this worker, yield the name, extended_worker_config_class (the adapters
+      # worker_config extended with this class's config attributes), and the default configuration values.
+      def each_config(worker_config_class, &block)
         # Configs are either defined with a define_configs call or default to a single instance with default_config
+        default_config = worker_config_class.initial_default_config
+        extended_worker_config_class = Class.new(worker_config_class)
+        self.bean_attrs.each do |args|
+          attr_method, name, type, description, default_value = args
+          extended_worker_config_class.send(attr_method, name, type, description, :config_item => true)
+          default_config[name.to_sym] = default_value
+        end
         if @configs
           @configs.each do |name, config|
-            yield name, default_config.merge(config)
+            yield name, extended_worker_config_class, default_config.merge(config)
           end
         else
-          yield default_name, default_config
+          yield default_name, extended_worker_config_class, default_config
         end
       end
-
-      #######
-      private
-      #######
-
-      def make_bean_attr(attr_method, name, type, description, default_value)
-        config_class.send(attr_method, name, type, description, :config_item => true)
-        default_config[name.to_sym] = default_value
-      end
     end
 
     def self.included(base)
@@ -73,10 +67,6 @@ module Qwirk
       @worker_classes ||= []
     end
 
-    def start
-      raise "Need to override start method in #{self.class.name}"
-    end
-
     def stop
       raise "Need to override stop method in #{self.class.name}"
     end