qwirk 0.1.0 → 0.2.0

data/History.md

@@ -1,6 +1,11 @@
 Qwirk Changelog
 =====================
 
+0.2.0
+-----
+
+ - Various fixes and improving of the shutdown logic.
+
 0.0.1
 -----
 

data/lib/qwirk.rb

@@ -7,8 +7,9 @@ module Qwirk
 
   DEFAULT_NAME = 'Qwirk'
 
-  @@config = nil
-  @@hash   = {}
+  @@config      = nil
+  @@environment = nil
+  @@hash        = {}
 
   class MyBean
     include Rumx::Bean
@@ -22,27 +23,48 @@ module Qwirk
   end
 
   def self.config=(config)
+    #if config.has_key?(:adapter)
     @@config = config
     Rumx::Bean.root.bean_add_child(DEFAULT_NAME, MyBean.new(@@hash))
   end
 
+  def self.environment=(environment)
+    @@environment = environment
+  end
+
+  def self.config_file=(config_file)
+    raise "No such file: #{config_file}" unless File.exist?(config_file)
+    config = YAML.load(ERB.new(File.read(config_file), nil, '-').result(binding))
+    config = config[@@environment] if config && @@environment
+    if config.has_key?(:adapter) || config.has_key?('adapter')
+      # Single level, one default adapter
+      @@config = {'default' => config}
+    else
+      @@config = config
+    end
+  end
+
   def self.[](adapter_key)
     if @@config.nil?
       if defined?(Rails)
-        # Allow user to use JMS w/o modifying qwirk.yml which could be checked in and hose other users
-        env = ENV['QWIRK_ENV'] || Rails.env
-        self.config = YAML.load(ERB.new(File.read(Rails.root.join("config", "qwirk.yml")), nil, '-').result(binding))[env]
+        # Allow user to use a different adapter w/o modifying qwirk.yml which could be checked in and hose other users
+        self.environment = ENV['QWIRK_ENV'] || Rails.env
+        self.config_file = Rails.root.join('config', 'qwirk.yml')
         Manager.default_options = {
-            :persist_file    => Rails.root.join('log', 'qwirk_persist.yml'),
+            :persist_file    => Rails.root.join('log',    'qwirk_persist.yml'),
             :worker_file     => Rails.root.join('config', 'qwirk_workers.yml'),
-            :stop_on_signal  => true,
-            :env             => env,
+            :env             => @@environment,
         }
       end
     end
     raise 'Qwirk not configured' unless @@config && @@config[adapter_key]
-    Qwirk.logger.debug {"Creating Qwirk::AdapterFactory key=#{adapter_key} config=#{@@config[adapter_key].inspect}"}
-    @@hash[adapter_key] ||= Qwirk::AdapterFactory.new(adapter_key, @@config[adapter_key])
+    @@hash[adapter_key] ||= begin
+      Qwirk.logger.debug {"Creating Qwirk::AdapterFactory key=#{adapter_key} config=#{@@config[adapter_key].inspect}"}
+      config = @@config[adapter_key]
+      raise "No config for key #{adapter_key}, keys=#{config.keys.inspect}" unless config
+      # Create the adapter, turning all the keys into symbols
+      Qwirk::AdapterFactory.new(adapter_key, Hash[config.map{|(k,v)| [k.to_sym,v]}])
+    end
   end
 
   def self.register_adapter(key, publisher_class, worker_config_class, &block)
@@ -52,6 +74,25 @@ module Qwirk
   def self.fail_queue_name(queue_name)
     return "#{queue_name.to_s}Fail"
   end
+
+  ## From here on down are proxies to the default adapter to keep the API simpler for setups with a single adapter
+  # TODO: Allow the setting of the default adapter
+
+  def self.create_publisher(options={})
+    self['default'].create_publisher(options)
+  end
+
+  def self.create_manager(options={})
+    self['default'].create_manager(options)
+  end
+
+  def self.create_publisher_impl(queue_name, topic_name, options, response_options)
+    self['default'].create_publisher_impl(queue_name, topic_name, options, response_options)
+  end
+
+  def self.in_process?
+    self['default'].in_process?
+  end
 end
 
 # We have to define the above before we define the adapters so the Qwirk#register_adapter call will work

data/lib/qwirk/adapter/base/expanding_worker_config.rb

@@ -31,7 +31,7 @@ module Qwirk
 
         def min_count=(new_min_count)
           return if @min_count == new_min_count
-          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped
+          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped?
           Qwirk.logger.info "#{self.worker_class.name}: Changing min number of workers from #{@min_count} to #{new_min_count}"
           self.max_count = new_min_count if @max_count < new_min_count
           @worker_mutex.synchronize do
@@ -42,34 +42,47 @@ module Qwirk
 
         def max_count=(new_max_count)
           return if @max_count == new_max_count
-          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped
+          raise "#{self.worker_class.name}-#{self.name}: Can't change count since we've been stopped" if self.stopped?
           Qwirk.logger.info "#{self.worker_class.name}: Changing max number of workers from #{@max_count} to #{new_max_count}"
           self.min_count = new_max_count if @min_count > new_max_count
           @min_count = 1 if @min_count == 0 && new_max_count > 0
+          deleted_workers = []
           @worker_mutex.synchronize do
             @timer ||= Rumx::Beans::TimerAndError.new
             if @workers.size > new_max_count
-              @workers[new_max_count..-1].each { |worker| worker.stop }
-              while @workers.size > new_max_count
-                @workers.last.stop
-                @worker_condition.wait(@worker_mutex)
-              end
+              deleted_workers = @workers[new_max_count..-1]
+              deleted_workers.each { |worker| worker.stop }
             end
             @max_count = new_max_count
           end
+          deleted_workers.each { |worker| worker.join }
         end
 
-        def stop
-          Qwirk.logger.debug { "In expanding_worker_config stop" }
-          # First stop the impl.  For InMemory, this will not return until all the messages in the queue have
-          # been processed since these messages are not persistent.
-          @impl.stop
-          @worker_mutex.synchronize do
-            @workers.each { |worker| worker.stop }
-            while @workers.size > 0
-              @worker_condition.wait(@worker_mutex)
+        # TODO: Need this?  Should I only be calling worker.stop when stopping individual workers?
+        #def stop
+        #  Qwirk.logger.debug { "#{self}: In expanding_worker_config stop" }
+        #  # First stop the impl.  For InMemory, this will not return until all the messages in the queue have
+        #  # been processed since these messages are not persistent.
+        #  @worker_mutex.synchronize do
+        #    @workers.each { |worker| worker.stop }
+        #    while @workers.size > 0
+        #      @worker_condition.wait(@worker_mutex)
+        #    end
+        #    super
+        #  end
+        #end
+
+        def join(timeout=nil)
+          workers = @worker_mutex.synchronize { @workers.dup }
+          if timeout
+            end_time = Time.now + timeout
+            workers.each do |worker|
+              t = end_time - Time.now
+              t = 0 if t < 0
+              worker.join(t)
             end
-            super
+          else
+            workers.each { |worker| worker.join }
           end
         end
 
@@ -103,7 +116,7 @@ module Qwirk
               #Qwirk.logger.debug { "#{self}: start=#{start_read_time} end=#{end_read_time} thres=#{@max_read_threshold} add_new_worker=#{add_new_worker}" }
               add_new_worker = false if (end_read_time - start_read_time) > @max_read_threshold
             end
-            add_worker if add_new_worker && @workers.size < @max_count
+            add_worker if !self.stopped? && add_new_worker && @workers.size < @max_count
           end
         end
 

data/lib/qwirk/adapter/base/worker_config.rb

@@ -8,10 +8,10 @@ module Qwirk
 
         # Make explicit the instance variables available to the derived adapter classes
         attr_reader        :adapter_factory, :name, :manager, :worker_class, :default_options, :options,
-                           :stopped, :queue_options, :response_options, :marshaler
+                           :queue_options, :response_options, :marshaler
         attr_accessor      :queue_name, :topic_name
 
-                           bean_attr_reader   :timer,     :bean,    'Track the times for this worker'
+        bean_attr_reader   :timer,     :bean,    'Track the times for this worker'
         bean_attr_accessor :log_times, :boolean, 'Log the times for this worker'
 
         # Define the default config values for the attributes all workers will share.  These will be sent as options to the constructor
@@ -23,18 +23,35 @@ module Qwirk
           :ruby
         end
 
+        # By default, workers do not run in the same process as the publishers
+        def self.in_process?(config)
+          false
+        end
+
         # Create new WorkerConfig to manage workers of a common class
         def initialize(adapter_factory, name, manager, worker_class, default_options, options)
           @adapter_factory  = adapter_factory
           @name             = name
           @manager          = manager
           @worker_class     = worker_class
-          @default_options  = default_options
-          @options          = options
-          @stopped          = false
-          @queue_name       = worker_class.queue_name(@name)
-          @topic_name       = worker_class.topic_name
-          @queue_options    = worker_class.queue_options
+          @default_options  = default_options.dup
+          @options          = options.dup
+          # First option is getting the queue or topic from the given options
+          @queue_name       = options.delete(:queue_name)
+          @topic_name       = !@queue_name && options.delete(:queue_name)
+          # Second option is getting them from the default options
+          @queue_name     ||= !@topic_name && default_options.delete(:queue_name)
+          @topic_name     ||= !@queue_name && default_options.delete(:topic_name)
+          # Third (and most likely) option is getting it based on the class topic or queue DSL or defaulting to the default queue
+          @queue_name     ||= !@topic_name && worker_class.queue_name(@name)
+          @topic_name     ||= !@queue_name && worker_class.topic_name
+
+          if @queue_name
+            more_queue_options = (default_options.delete(:queue_options) || {}).merge(options.delete(:queue_options) || {})
+          else
+            more_queue_options = (default_options.delete(:topic_options) || {}).merge(options.delete(:topic_options) || {})
+          end
+          @queue_options    = worker_class.queue_options.merge(more_queue_options)
           @response_options = @queue_options[:response] || {}
 
           # Defines how we will marshal the response
@@ -53,7 +70,7 @@ module Qwirk
               begin
                 instance_variable_set("@#{key}", value)
               rescue Exception => e
-                Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, default assignment of #{key}=#{value} was invalid"
+                Qwirk.logger.debug "DEBUG: During initialization of #{worker_class.name} config=#{@name}, default assignment of #{key}=#{value} was ignored"
               end
             end
           end
@@ -62,7 +79,7 @@ module Qwirk
             begin
               send(key.to_s+'=', value)
             rescue Exception => e
-              Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, assignment of #{key}=#{value} was invalid"
+              Qwirk.logger.debug "DEBUG: During initialization of #{worker_class.name} config=#{@name}, assignment of #{key}=#{value} was ignored"
             end
           end
         end
@@ -72,7 +89,13 @@ module Qwirk
         end
 
         def stop
-          @stopped = true
+        end
+
+        def join(timeout=nil)
+        end
+
+        def stopped?
+          self.manager.stopped?
         end
 
         def worker_stopped(worker)

data/lib/qwirk/adapter/in_memory/publisher.rb

@@ -85,9 +85,9 @@ module Qwirk
           end
 
           def stop
+            # TODO - Need to think about what should actually occur here?
             return if @stopped
             Qwirk.logger.info "Stopping Task worker #{@consumer_queue}"
-            # Don't clobber the session before a reply
             @producer_queue.interrupt_read
             @stopped = true
           end

data/lib/qwirk/adapter/in_memory/queue.rb

@@ -9,14 +9,13 @@ module Qwirk
         attr_accessor :name, :max_size
 
         def initialize(name)
-          @name                   = name
-          @max_size               = 0
-          @outstanding_hash_mutex = Mutex.new
-          @read_condition         = ConditionVariable.new
-          @write_condition        = ConditionVariable.new
-          @stop_condition         = ConditionVariable.new
-          @array                  = []
-          @stopped                = false
+          @name            = name
+          @max_size        = 0
+          @array_mutex     = Mutex.new
+          @read_condition  = ConditionVariable.new
+          @write_condition = ConditionVariable.new
+          @array           = []
+          @stopping        = false
         end
 
         def size
@@ -24,47 +23,45 @@ module Qwirk
         end
 
         def stop
-          return if @stopped
-          @stopped = true
-          @outstanding_hash_mutex.synchronize do
+          return if @stopping
+          @stopping = true
+          @array_mutex.synchronize do
             @write_condition.broadcast
-            until @array.empty?
-              @stop_condition.wait(@outstanding_hash_mutex)
-            end
             @read_condition.broadcast
           end
         end
 
+        def stopped?
+          @array_mutex.synchronize do
+            return @stopping && @array.empty?
+          end
+        end
+
         def interrupt_read
-          @outstanding_hash_mutex.synchronize do
+          @array_mutex.synchronize do
             @read_condition.broadcast
           end
         end
 
-        # Block read until a message or we get stopped.  stoppable is an object that responds to stopped (a worker or some kind of consumer)
+        # Block read until a message or we get stopped.
         def read(stoppable)
-          @outstanding_hash_mutex.synchronize do
-            until @stopped  || stoppable.stopped do
-              unless @array.empty?
+          @array_mutex.synchronize do
+            until stoppable.stopped || (@stopping  && @array.empty?)
+              if @array.empty?
+                @read_condition.wait(@array_mutex)
+              else
                 @write_condition.signal
                 return @array.shift
               end
-              @read_condition.wait(@outstanding_hash_mutex)
-            end
-            return if stoppable.stopped
-            # We're not persistent, so even though we're stopped we're going to allow our stoppables to keep reading until the queue's empty
-            unless @array.empty?
-              @stop_condition.signal
-              return @array.shift
             end
           end
           return nil
         end
 
         def write(obj)
-          @outstanding_hash_mutex.synchronize do
+          @array_mutex.synchronize do
             # We just drop the message if no workers have been configured yet
-            while !@stopped
+            while !@stopping
               if @max_size == 0
                 Qwirk.logger.warn "No worker for queue #{@name}, dropping message #{obj.inspect}"
                 return
@@ -75,8 +72,9 @@ module Qwirk
                 return
               end
               # TODO: Let's allow various write_full_modes such as :block, :remove_oldest, ? (Currently only blocks)
-              @write_condition.wait(@outstanding_hash_mutex)
+              @write_condition.wait(@array_mutex)
             end
+            Qwirk.logger.warn "Queue has been stopped #{@name}, dropping message #{obj.inspect}"
           end
         end
 

data/lib/qwirk/adapter/in_memory/topic.rb

@@ -4,14 +4,13 @@ module Qwirk
 
       class Topic
         def initialize(name)
-          @name            = name
-          @outstanding_hash_mutex           = Mutex.new
-          @worker_hash     = {}
-          @stopped         = false
+          @name              = name
+          @worker_hash_mutex = Mutex.new
+          @worker_hash       = {}
         end
 
         def get_worker_queue(worker_name, queue_max_size)
-          @outstanding_hash_mutex.synchronize do
+          @worker_hash_mutex.synchronize do
             queue = @worker_hash[worker_name] ||= Queue.new("#{@name}:#{worker_name}")
             queue.max_size = queue_max_size
             return queue
@@ -19,9 +18,10 @@ module Qwirk
         end
 
         def stop
-          @stopped = true
-          @worker_hash.each_value do |queue|
-            queue.stop
+          @worker_hash_mutex.synchronize do
+            @worker_hash.each_value do |queue|
+              queue.stop
+            end
           end
         end
 
@@ -30,11 +30,9 @@ module Qwirk
         end
 
         def write(obj)
-          @outstanding_hash_mutex.synchronize do
+          @worker_hash_mutex.synchronize do
             @worker_hash.each_value do |queue|
-              if !@stopped
-                queue.write(obj)
-              end
+              queue.write(obj)
             end
           end
         end

data/lib/qwirk/adapter/in_memory/worker.rb

@@ -46,6 +46,15 @@ module Qwirk
           @queue.interrupt_read
         end
 
+        # If the worker_config has been commanded to stop, workers will continue processing messages until this returns true
+        def ready_to_stop?
+          @queue.stopped?
+        end
+
+        def to_s
+          "#{@name} (InMemory)"
+        end
+
         ## End of required override methods for worker impl
         private
 

data/lib/qwirk/adapter/in_memory/worker_config.rb

@@ -16,6 +16,10 @@ module Qwirk
           super.merge(:queue_max_size => 100)
         end
 
+        def self.in_process?(config)
+          true
+        end
+
         def init
           super
           @queue = Factory.get_worker_queue(self.name, self.queue_name, self.topic_name, @queue_max_size)