qwirk 0.0.1 → 0.1.0

data/README.md

@@ -4,23 +4,18 @@ http://github.com/ClarityServices/qwirk
 
 ## Description:
 
-JRuby library for performing background tasks via JMS.
-
+Library for performing background tasks as well as asynchronous and parallel RPC calls
 Alpha version.  API still subject to change.
 
-## Features/Problems:
-
-Currently tested only for ActiveMQ
-
 ## Install:
 
   gem install qwirk
 
 ## Rails Usage:
 
-TODO: This stuff is all obsolete modern_times documentation.  Look at the examples for usage information.
+TODO: This stuff is all obsolete modern_times documentation.  Look at the project http://github.com/ClarityServices/qwirk_examples for usage information.
 
-Create config/jms.yml which might look as follows:
+Create config/qwirk.yml which might look as follows:
 
     development_server: &defaults
       :factory: org.apache.activemq.ActiveMQConnectionFactory
@@ -98,7 +93,7 @@ examples for registering marshal strategies):
 
     Qwirk.init_rails
     # Publishers can be defined wherever appropriate, probably as class variables within the class that uses it
-    $foo_publisher = Qwirk::QueueAdapter::JMS::Publisher.new('Foo')
+    $foo_publisher = Qwirk::Adapter::JMS::Publisher.new('Foo')
 
 When creating publishers, you will probably want to store the value in a class variable.  Publishers internally
 make use of a session pool for communicating with the JMS server so you wouldn't want to create a new connection
@@ -111,7 +106,7 @@ In your code, queue foo objects:
 In app/workers, create a FooWorker class:
 
     class FooWorker
-      include Qwirk::QueueAdapter::JMS::Worker
+      include Qwirk::Adapter::JMS::Worker
       def perform(my_foo_object)
         # Operate on my_foo_object
       end
@@ -137,7 +132,7 @@ receives all messages instead of a group of workers (threads) collectively recei
 problem). For instance, suppose you have the following workers:
 
     class FooWorker
-      include Qwirk::QueueAdapter::JMS::Worker
+      include Qwirk::Adapter::JMS::Worker
       virtual_topic 'inquiry'
 
       def perform(my_inquiry)
@@ -146,7 +141,7 @@ problem). For instance, suppose you have the following workers:
     end
 
     class BarWorker
-      include Qwirk::QueueAdapter::JMS::Worker
+      include Qwirk::Adapter::JMS::Worker
       virtual_topic 'inquiry'
 
       def perform(my_inquiry)
@@ -156,7 +151,7 @@ problem). For instance, suppose you have the following workers:
 
 Then you can create a publisher where messages are delivered to both workers:
 
-    @@publisher = Qwirk::QueueAdapter::JMS::Publisher.new(:virtual_topic_name => 'inquiry')
+    @@publisher = Qwirk::Adapter::JMS::Publisher.new(:virtual_topic_name => 'inquiry')
     ...
     @@publisher.publish(my_inquiry)
 
@@ -166,7 +161,7 @@ Then you can create a publisher where messages are delivered to both workers:
 TODO: See examples/requestor
 
 
-## Requestor Pattern with Multiple RequestWorkers:
+## Requestor Pattern with Multiple ReplyWorkers:
 
 TODO: See examples/advanced_requestor
 

data/lib/qwirk.rb

@@ -1,17 +1,5 @@
 require 'rubygems'
-require 'qwirk/remote_exception'
-require 'qwirk/marshal_strategy'
-require 'qwirk/base_worker'
-require 'qwirk/worker_config'
-require 'qwirk/worker'
-require 'qwirk/request_worker'
-require 'qwirk/task'
-require 'qwirk/publisher'
-require 'qwirk/publish_handle'
-require 'qwirk/adapter'
-require 'qwirk/queue_adapter'
-#require 'qwirk/batch'
-require 'qwirk/manager'
+require 'rumx'
 require 'qwirk/loggable'
 
 module Qwirk
@@ -25,7 +13,8 @@ module Qwirk
   class MyBean
     include Rumx::Bean
 
-    bean_attr_reader :adapters,  :hash,   'Adapters', :hash_type => :bean
+    # These are actually AdapterFactory's but presenting as adapters to the user.
+    bean_attr_reader :adapters,  :hash, 'Adapters', :hash_type => :bean
 
     def initialize(hash)
       @adapters = hash
@@ -37,7 +26,7 @@ module Qwirk
     Rumx::Bean.root.bean_add_child(DEFAULT_NAME, MyBean.new(@@hash))
   end
 
-  def self.[](key)
+  def self.[](adapter_key)
     if @@config.nil?
       if defined?(Rails)
         # Allow user to use JMS w/o modifying qwirk.yml which could be checked in and hose other users
@@ -51,8 +40,13 @@ module Qwirk
         }
       end
     end
-    raise 'Qwirk not configured' unless @@config && @@config[key]
-    @@hash[key] ||= Qwirk::Adapter.new(@@config[key])
+    raise 'Qwirk not configured' unless @@config && @@config[adapter_key]
+    Qwirk.logger.debug {"Creating Qwirk::AdapterFactory key=#{adapter_key} config=#{@@config[adapter_key].inspect}"}
+    @@hash[adapter_key] ||= Qwirk::AdapterFactory.new(adapter_key, @@config[adapter_key])
+  end
+
+  def self.register_adapter(key, publisher_class, worker_config_class, &block)
+    AdapterFactory.register(key, publisher_class, worker_config_class, &block)
   end
 
   def self.fail_queue_name(queue_name)
@@ -60,4 +54,19 @@ module Qwirk
   end
 end
 
+# We have to define the above before we define the adapters so the Qwirk#register_adapter call will work
+# AdapterFactory is also required before adapter for this reason.
+require 'qwirk/adapter_factory'
+require 'qwirk/adapter'
+require 'qwirk/base_worker'
+#require 'qwirk/batch'
+require 'qwirk/manager'
+require 'qwirk/marshal_strategy'
+require 'qwirk/publish_handle'
+require 'qwirk/publisher'
+require 'qwirk/remote_exception'
+require 'qwirk/task'
+require 'qwirk/worker'
+require 'qwirk/reply_worker'
+
 require 'qwirk/engine' if defined?(Rails)

data/lib/qwirk/adapter.rb

@@ -1,45 +1,3 @@
-module Qwirk
-
-  # Defines the queuing strategy.  Currently, only JMS and InMem.
-  class Adapter
-    include Rumx::Bean
-
-    attr_reader :config, :log_times, :adapter_info
-
-    def initialize(config)
-      @config = config.dup
-      @log_times = config.delete(:log_times)
-
-      adapter = config.delete(:adapter)
-      raise "No adapter definition" unless adapter
-      namespace = Qwirk::QueueAdapter.const_get(adapter)
-      @adapter_info = nil
-      if namespace.respond_to?(:init)
-        @adapter_info = namespace.init(config)
-      end
-      @publisher_klass     = namespace.const_get('Publisher')
-      @worker_config_klass = namespace.const_get('WorkerConfig')
-    end
-
-    def create_publisher(options={})
-      @publisher_parent ||= Rumx::Beans::Folder.new
-      publisher = Publisher.new(self, @config.merge(options))
-      @publisher_parent.bean_add_child(publisher.to_s, publisher)
-      return publisher
-    end
-
-    def create_manager(options={})
-      manager = Manager.new(self, @config.merge(options))
-      bean_add_child(:Workers, manager)
-      return manager
-    end
-
-    def create_adapter_publisher(queue_name, topic_name, options, response_options)
-      @publisher_klass.new(self, queue_name, topic_name, options, response_options)
-    end
-
-    def create_adapter_worker_config(parent, queue_name, topic_name, options, response_options)
-      @worker_config_klass.new(self, parent, queue_name, topic_name, options, response_options)
-    end
-  end
-end
+require 'qwirk/adapter/base'
+require 'qwirk/adapter/inline'
+require 'qwirk/adapter/in_memory'

data/lib/qwirk/adapter/base.rb

@@ -0,0 +1,2 @@
+require 'qwirk/adapter/base/worker_config'
+require 'qwirk/adapter/base/expanding_worker_config'

data/lib/qwirk/adapter/base/expanding_worker_config.rb

@@ -0,0 +1,133 @@
+module Qwirk
+  module Adapter
+    module Base
+      class ExpandingWorkerConfig < WorkerConfig
+
+        bean_reader        :count,               :integer, 'Current number of workers'
+        bean_attr_accessor :min_count,           :integer, 'Min number of workers allowed', :config_item => true
+        bean_attr_accessor :max_count,           :integer, 'Max number of workers allowed', :config_item => true
+        bean_attr_accessor :idle_worker_timeout, :integer, 'Timeout where an idle worker will be removed from the worker pool and it\'s resources closed (0 for no removal)', :config_item => true
+        bean_attr_accessor :max_read_threshold,  :float,   'Threshold where a new worker will be added if none of the workers have had to wait this amount of time on a read', :config_item => true
+
+        # Define the default config values for the attributes all workers will share.  These will be sent as options to the constructor
+        def self.initial_default_config
+          super.merge(:min_count => 0, :max_count => 0, :idle_worker_timeout => 60, :max_read_threshold => 1.0)
+        end
+
+        def init
+          super
+          @workers          = []
+          @min_count        = 0
+          @max_count        = 0
+          @index_count      = 0
+          @index_mutex      = Mutex.new
+          @worker_mutex     = Mutex.new
+          @worker_condition = ConditionVariable.new
+        end
+
+        def count
+          @worker_mutex.synchronize { return @workers.size }
+        end
+
+        def min_count=(new_min_count)
+          return if @min_count == new_min_count
+          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped
+          Qwirk.logger.info "#{self.worker_class.name}: Changing min number of workers from #{@min_count} to #{new_min_count}"
+          self.max_count = new_min_count if @max_count < new_min_count
+          @worker_mutex.synchronize do
+            add_worker while @workers.size < new_min_count
+            @min_count = new_min_count
+          end
+        end
+
+        def max_count=(new_max_count)
+          return if @max_count == new_max_count
+          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped
+          Qwirk.logger.info "#{self.worker_class.name}: Changing max number of workers from #{@max_count} to #{new_max_count}"
+          self.min_count = new_max_count if @min_count > new_max_count
+          @min_count = 1 if @min_count == 0 && new_max_count > 0
+          @worker_mutex.synchronize do
+            @timer ||= Rumx::Beans::TimerAndError.new
+            if @workers.size > new_max_count
+              @workers[new_max_count..-1].each { |worker| worker.stop }
+              while @workers.size > new_max_count
+                @workers.last.stop
+                @worker_condition.wait(@worker_mutex)
+              end
+            end
+            @max_count = new_max_count
+          end
+        end
+
+        def stop
+          Qwirk.logger.debug { "In expanding_worker_config stop" }
+          # First stop the impl.  For InMemory, this will not return until all the messages in the queue have
+          # been processed since these messages are not persistent.
+          @impl.stop
+          @worker_mutex.synchronize do
+            @workers.each { |worker| worker.stop }
+            while @workers.size > 0
+              @worker_condition.wait(@worker_mutex)
+            end
+            super
+          end
+        end
+
+        def worker_stopped(worker)
+          remove_worker(worker)
+        end
+
+        def periodic_call(poll_time)
+          now = Time.now
+          add_new_worker = true
+          worker_stopped = false
+          @worker_mutex.synchronize do
+            # reverse_each to remove later workers first
+            @workers.reverse_each do |worker|
+              start_worker_time = worker.start_worker_time
+              start_read_time = worker.start_read_time
+              if !start_read_time || (now - start_worker_time) < (poll_time + @max_read_threshold)
+                #Qwirk.logger.debug { "#{self}: Skipping newly created worker" }
+                add_new_worker = false
+                next
+              end
+              end_read_time = worker.start_processing_time
+              # If the processing time is actually from the previous processing, then we're probably still waiting for the read to complete.
+              if !end_read_time || end_read_time < start_read_time
+                if !worker_stopped && @workers.size > @min_count && (now - start_read_time) > @idle_worker_timeout
+                  worker.stop
+                  worker_stopped = true
+                end
+                end_read_time = now
+              end
+              #Qwirk.logger.debug { "#{self}: start=#{start_read_time} end=#{end_read_time} thres=#{@max_read_threshold} add_new_worker=#{add_new_worker}" }
+              add_new_worker = false if (end_read_time - start_read_time) > @max_read_threshold
+            end
+            add_worker if add_new_worker && @workers.size < @max_count
+          end
+        end
+
+        private
+
+        def add_worker
+          worker = self.worker_class.new
+          worker.init(@index_count, self)
+          worker.start
+          Qwirk.logger.debug {"#{self}: Adding worker #{worker}"}
+          @index_mutex.synchronize { @index_count += 1 }
+          @workers << worker
+        rescue Exception => e
+          Qwirk.logger.error("Unable to add #{self.worker_class} worker: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+        end
+
+        def remove_worker(worker)
+          Qwirk.logger.debug {"#{self}: Deleting worker #{worker}"}
+          @worker_mutex.synchronize do
+            @workers.delete(worker)
+            @worker_condition.broadcast
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter/base/worker_config.rb

@@ -0,0 +1,104 @@
+require 'rumx'
+
+module Qwirk
+  module Adapter
+    module Base
+      class WorkerConfig
+        include Rumx::Bean
+
+        # Make explicit the instance variables available to the derived adapter classes
+        attr_reader        :adapter_factory, :name, :manager, :worker_class, :default_options, :options,
+                           :stopped, :queue_options, :response_options, :marshaler
+        attr_accessor      :queue_name, :topic_name
+
+                           bean_attr_reader   :timer,     :bean,    'Track the times for this worker'
+        bean_attr_accessor :log_times, :boolean, 'Log the times for this worker'
+
+        # Define the default config values for the attributes all workers will share.  These will be sent as options to the constructor
+        def self.initial_default_config
+          {}
+        end
+
+        def self.default_marshal_sym
+          :ruby
+        end
+
+        # Create new WorkerConfig to manage workers of a common class
+        def initialize(adapter_factory, name, manager, worker_class, default_options, options)
+          @adapter_factory  = adapter_factory
+          @name             = name
+          @manager          = manager
+          @worker_class     = worker_class
+          @default_options  = default_options
+          @options          = options
+          @stopped          = false
+          @queue_name       = worker_class.queue_name(@name)
+          @topic_name       = worker_class.topic_name
+          @queue_options    = worker_class.queue_options
+          @response_options = @queue_options[:response] || {}
+
+          # Defines how we will marshal the response
+          marshal_sym       = (response_options[:marshal] || self.class.default_marshal_sym)
+          @marshaler        = MarshalStrategy.find(marshal_sym)
+          @log_times        = adapter_factory.log_times
+
+          init
+
+          #Qwirk.logger.debug { "options=#{options.inspect}" }
+          default_options.each do |key, value|
+            begin
+              send(key.to_s+'=', value)
+            rescue Exception => e
+              # Let config_reader's set a default value
+              begin
+                instance_variable_set("@#{key}", value)
+              rescue Exception => e
+                Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, default assignment of #{key}=#{value} was invalid"
+              end
+            end
+          end
+          # Run the specified options after the default options, so that codependant options don't get overwritten (like min_count/max_count)
+          options.each do |key, value|
+            begin
+              send(key.to_s+'=', value)
+            rescue Exception => e
+              Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, assignment of #{key}=#{value} was invalid"
+            end
+          end
+        end
+
+        # Allow extensions to initialize before setting the attributes
+        def init
+        end
+
+        def stop
+          @stopped = true
+        end
+
+        def worker_stopped(worker)
+        end
+
+        # Override rumx bean method
+        def bean_attributes_changed
+          super
+          @manager.save_persist_state
+        end
+
+        def marshal_response(object)
+          @marshaler.marshal(object)
+        end
+
+        def unmarshal_response(marshaled_object)
+          @marshaler.unmarshal(marshaled_object)
+        end
+
+        def periodic_call(poll_time)
+        end
+
+        def to_s
+          @name
+        end
+      end
+    end
+  end
+end

data/lib/qwirk/adapter/in_memory.rb

@@ -0,0 +1,13 @@
+require 'qwirk/adapter/in_memory/factory'
+require 'qwirk/adapter/in_memory/publisher'
+require 'qwirk/adapter/in_memory/queue'
+require 'qwirk/adapter/in_memory/reply_queue'
+require 'qwirk/adapter/in_memory/topic'
+require 'qwirk/adapter/in_memory/worker_config'
+require 'qwirk/adapter/in_memory/worker'
+
+::Qwirk.register_adapter(
+    :in_memory,
+    ::Qwirk::Adapter::InMemory::Publisher,
+    ::Qwirk::Adapter::InMemory::WorkerConfig
+)