qwirk 0.0.1

data/lib/qwirk/version.rb

@@ -0,0 +1,3 @@
+module Qwirk
+  VERSION = '0.0.1'
+end

data/lib/qwirk/worker.rb

@@ -0,0 +1,222 @@
+module Qwirk
+
+  # Base Worker Class for any class that will be processing messages from topics or queues
+  # By default, it will consume messages from a queue with the class name minus the Worker postfix.
+  # For example, the queue call is unnecessary as it will default to a value of 'Foo' anyways:
+  #  class FooWorker
+  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    queue 'Foo'
+  #    def perform(obj)
+  #      # Perform work on obj
+  #    end
+  #  end
+  #
+  # A topic can also be specified.  Note that for JMS, this is only supported under ActiveMQ.  On others,
+  # each thread for a given worker will act as a separate subscriber.
+  # (For ActiveMQ - see http://activemq.apache.org/virtual-destinations.html):
+  #  class FooWorker
+  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    topic 'Zulu'
+  #    def perform(obj)
+  #      # Perform work on obj
+  #    end
+  #  end
+  #
+  # TODO (maybe):
+  # Filters can also be specified within the class:
+  #  class FooWorker
+  #    include Qwirk::QueueAdapter::JMS::Worker
+  #    filter 'age > 30'
+  #    def perform(obj)
+  #      # Perform work on obj
+  #    end
+  #  end
+  #
+  #
+  module Worker
+    include Qwirk::BaseWorker
+
+    attr_accessor :message
+    attr_reader   :status, :adapter, :start_worker_time, :start_read_time, :start_processing_time
+
+    module ClassMethods
+      def queue(name, opts={})
+        # If we're using the default name but we still want to set queue options, then a name won't be given.
+        if name.kind_of?(Hash)
+          @queue_options = name
+        else
+          @queue_name = name.to_s
+          @queue_options = opts
+        end
+      end
+
+      def topic(name, options={})
+        @topic_name = name.to_s
+        @queue_options = options
+      end
+
+      # Set the fail_queue
+      # target =>
+      #   boolean
+      #     true - exceptions in the worker will cause the message to be forwarded to the queue of <default-name>Fail
+      #       For instance, an Exception in FooWorker#perform will forward the message to the queue FooFail
+      #     false - exceptions will not result in the message being forwarded to a fail queue
+      #   string - equivalent to true but the string defines the name of the fail queue
+      def fail_queue(target, opts={})
+        @fail_queue_target = target
+      end
+
+      def fail_queue_target
+        @fail_queue_target
+      end
+
+      # Defines the default value of the fail_queue_target.  For extenders of this class, the default will be true
+      # but extenders can change this (RequestWorker returns exceptions to the caller so it defaults to false).
+      def default_fail_queue_target
+        true
+      end
+
+      def queue_name(default_name)
+        puts "getting queue_name queue=#{@queue_name} topic=#{@topic_name} default=#{default_name}"
+        return @queue_name if @queue_name
+        return nil if @topic_name
+        return default_name
+      end
+
+      def topic_name
+        @topic_name
+      end
+
+      def queue_options
+        @queue_options ||= {}
+      end
+
+      def fail_queue_name(worker_config)
+        # TBD - Set up fail_queue as a config
+        target = self.class.fail_queue_target
+        # Don't overwrite if the user set to false, only if it was never set
+        target = self.class.default_fail_queue_target if target.nil?
+        if target == true
+          return Qwirk.fail_queue_name(config.name)
+        elsif target == false
+          return nil
+        elsif target.kind_of?(String)
+          return target
+        else
+          raise "Invalid fail queue: #{target}"
+        end
+      end
+    end
+
+    def self.included(base)
+      Qwirk::BaseWorker.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def start(index, worker_config)
+      @status               = 'Started'
+      @stopped              = false
+      @processing_mutex     = Mutex.new
+      self.index            = index
+      self.config           = worker_config
+      @adapter              = worker_config.adapter.create_worker
+      self.thread = Thread.new do
+        java.lang.Thread.current_thread.name = "Qwirk worker: #{self}" if RUBY_PLATFORM == 'jruby'
+        #Qwirk.logger.debug "#{worker}: Started thread with priority #{Thread.current.priority}"
+        event_loop
+      end
+    end
+
+    # Workers will be starting and stopping on an as needed basis.  Thus, when they get a stop command they should
+    # clean up any resources.  We don't want to clobber resources while a message is being processed so processing_mutex will surround
+    # message processessing and worker closing.
+    # From a JMS perspective, stop all workers (close consumer and session), stop the config.
+    # From an InMem perspective, we don't want the workers stopping until all messages in the queue have been processed.
+    # Therefore we want to stop the
+    def stop
+      puts "#{self}: In base worker stop"
+      @status  = 'Stopping'
+      @stopped = true
+      @processing_mutex.synchronize do
+        # This should interrupt @adapter.receive_message above and cause it to return nil
+        @adapter.stop
+      end
+      puts "#{self}: base worker stop complete"
+    end
+
+    def perform(object)
+      raise "#{self}: Need to override perform method in #{self.class.name} in order to act on #{object}"
+    end
+
+    def to_s
+      "#{config.name}:#{index}"
+    end
+
+    # Allow override of backtrace logging in case the client doesn't want to get spammed with it (maybe just config instead?)
+    def log_backtrace(e)
+      Qwirk.logger.error "\t#{e.backtrace.join("\n\t")}"
+    end
+
+    #########
+    protected
+    #########
+
+    # Start the event loop for handling messages off the queue
+    def event_loop
+      Qwirk.logger.debug "#{self}: Starting receive loop"
+      @start_worker_time = Time.now
+      while !@stopped && !config.adapter.stopped
+        puts "#{self}: Waiting for read"
+        @start_read_time = Time.now
+        msg = @adapter.receive_message
+        if msg
+          @start_processing_time = Time.now
+          Qwirk.logger.debug {"#{self}: Done waiting for read in #{@start_processing_time - @start_read_time} seconds"}
+          delta = config.timer.measure do
+            @processing_mutex.synchronize do
+              on_message(msg)
+              @adapter.acknowledge_message(msg)
+            end
+          end
+          Qwirk.logger.info {"#{self}::on_message (#{'%.1f' % delta}ms)"} if self.config.log_times
+          Qwirk.logger.flush if Qwirk.logger.respond_to?(:flush)
+        end
+      end
+      Qwirk.logger.info "#{self}: Exiting"
+    rescue Exception => e
+      @status = "Terminated: #{e.message}"
+      Qwirk.logger.error "#{self}: Exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+    ensure
+      @status = 'Stopped'
+      # TODO: necessary?
+      @adapter.stop
+      Qwirk.logger.flush if Qwirk.logger.respond_to?(:flush)
+      config.worker_stopped(self)
+    end
+
+    def on_message(message)
+      # TBD - Is it necessary to provide underlying message to worker?  Should we generically provide access to message attributes?  Do filters somehow fit in here?
+      @message = message
+      object = @adapter.message_to_object(message)
+      Qwirk.logger.debug {"#{self}: Received Object: #{object}"}
+      perform(object)
+    rescue Exception => e
+      on_exception(e)
+    ensure
+      Qwirk.logger.debug {"#{self}: Finished processing message"}
+    end
+
+    def on_exception(e)
+      Qwirk.logger.error "#{self}: Messaging Exception: #{e.message}"
+      log_backtrace(e)
+      @adapter.handle_failure(message, @fail_queue_name) if @fail_queue_name
+    rescue Exception => e
+      Qwirk.logger.error "#{self}: Exception in exception reply: #{e.message}"
+      log_backtrace(e)
+    end
+
+    def fail_queue_name
+      @fail_queue_name
+    end
+  end
+end

data/lib/qwirk/worker_config.rb

@@ -0,0 +1,187 @@
+require 'rumx'
+
+module Qwirk
+  class WorkerConfig
+    include Rumx::Bean
+
+    attr_reader        :name, :marshaler
+
+    bean_reader        :count,               :integer, 'Current number of workers'
+    bean_attr_accessor :min_count,           :integer, 'Min number of workers allowed', :config_item => true
+    bean_attr_accessor :max_count,           :integer, 'Max number of workers allowed', :config_item => true
+    bean_attr_accessor :idle_worker_timeout, :integer, 'Timeout where an idle worker will be removed from the worker pool and it\'s resources closed (0 for no removal)', :config_item => true
+    bean_attr_accessor :max_read_threshold,  :float,   'Threshold where a new worker will be added if none of the workers have had to wait this amount of time on a read', :config_item => true
+    # The adapter refers to the corresponding class in Qwirk::QueueAdapter::<type>::WorkerConfig
+    bean_attr_reader   :adapter,             :bean,    'Adapter for worker queue interface'
+    bean_attr_reader   :timer,               :bean,    'Track the times for this worker'
+    bean_attr_accessor :log_times,           :boolean, 'Log the times for this worker'
+
+    # Define the default config values for the attributes all workers will share.  These will be sent as options to the constructor
+    def self.initial_default_config
+      {:min_count => 0, :max_count => 0, :idle_worker_timeout => 60, :max_read_threshold => 1.0}
+    end
+
+    # Create new WorkerConfig to manage workers of a common class
+    def initialize(queue_adapter, name, manager, worker_class, default_options, options)
+      @name             = name
+      @manager          = manager
+      @worker_class     = worker_class
+      @workers          = []
+      @stopped          = false
+      @min_count        = 0
+      @max_count        = 0
+      @index_count      = 0
+      @index_mutex      = Mutex.new
+      @worker_mutex     = Mutex.new
+      @worker_condition = ConditionVariable.new
+      response_options  = worker_class.queue_options[:response] || {}
+      @adapter          = queue_adapter.create_adapter_worker_config(self, worker_class.queue_name(@name), worker_class.topic_name, worker_class.queue_options, response_options)
+      # Defines how we will marshal the response
+      marshal_sym       = (response_options[:marshal] || @adapter.default_marshal_sym)
+      @marshaler        = MarshalStrategy.find(marshal_sym)
+      @log_times        = queue_adapter.log_times
+
+      #Qwirk.logger.debug { "options=#{options.inspect}" }
+      default_options.each do |key, value|
+        begin
+          send(key.to_s+'=', value)
+        rescue Exception => e
+          # Let config_reader's set a default value
+          begin
+            instance_variable_set("@#{key}", value)
+          rescue Exception => e
+            Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, default assignment of #{key}=#{value} was invalid"
+          end
+        end
+      end
+      # Run the specified options after the default options, so that codependant options don't get overwritten (like min_count/max_count)
+      options.each do |key, value|
+        begin
+          send(key.to_s+'=', value)
+        rescue Exception => e
+          Qwirk.logger.warn "WARNING: During initialization of #{worker_class.name} config=#{@name}, assignment of #{key}=#{value} was invalid"
+        end
+      end
+    end
+
+    def count
+      @worker_mutex.synchronize { return @workers.size }
+    end
+
+    def min_count=(new_min_count)
+      return if @min_count == new_min_count
+      raise "#{@worker_class.name}-#{@name}: Can't change count since we've been stopped" if @stopped
+      Qwirk.logger.info "#{@worker_class.name}: Changing min number of workers from #{@min_count} to #{new_min_count}"
+      self.max_count = new_min_count if @max_count < new_min_count
+      @worker_mutex.synchronize do
+        add_worker while @workers.size < new_min_count
+        @min_count = new_min_count
+      end
+    end
+
+    def max_count=(new_max_count)
+      return if @max_count == new_max_count
+      raise "#{@worker_class.name}-#{@name}: Can't change count since we've been stopped" if @stopped
+      Qwirk.logger.info "#{@worker_class.name}: Changing max number of workers from #{@max_count} to #{new_max_count}"
+      self.min_count = new_max_count if @min_count > new_max_count
+      @min_count = 1 if @min_count == 0 && new_max_count > 0
+      @worker_mutex.synchronize do
+        @timer ||= Rumx::Beans::TimerAndError.new
+        if @workers.size > new_max_count
+          @workers[new_max_count..-1].each { |worker| worker.stop }
+          while @workers.size > new_max_count
+            @workers.last.stop
+            @worker_condition.wait(@worker_mutex)
+          end
+        end
+        @max_count = new_max_count
+      end
+    end
+
+    def stop
+      Qwirk.logger.debug { "In Base worker_config stop" }
+      # First stop the adapter.  For InMem, this will not return until all the messages in the queue have
+      # been processed since these messages are not persistent.
+      @adapter.stop
+      @worker_mutex.synchronize do
+        @workers.each { |worker| worker.stop }
+        while @workers.size > 0
+          @worker_condition.wait(@worker_mutex)
+        end
+        @stopped = true
+      end
+    end
+
+    def worker_stopped(worker)
+      remove_worker(worker)
+    end
+
+    # Override rumx bean method
+    def bean_attributes_changed
+      super
+      @manager.save_persist_state
+    end
+
+    def marshal_response(object)
+      @marshaler.marshal(object)
+    end
+
+    def unmarshal_response(marshaled_object)
+      @marshaler.unmarshal(marshaled_object)
+    end
+
+    def periodic_call(poll_time)
+      now = Time.now
+      add_new_worker = true
+      worker_stopped = false
+      @worker_mutex.synchronize do
+        # reverse_each to remove later workers first
+        @workers.reverse_each do |worker|
+          start_worker_time = worker.start_worker_time
+          start_read_time = worker.start_read_time
+          if !start_read_time || (now - start_worker_time) < (poll_time + @max_read_threshold)
+            #Qwirk.logger.debug { "#{self}: Skipping newly created worker" }
+            add_new_worker = false
+            next
+          end
+          end_read_time = worker.start_processing_time
+          # If the processing time is actually from the previous processing, then we're probably still waiting for the read to complete.
+          if !end_read_time || end_read_time < start_read_time
+            if !worker_stopped && @workers.size > @min_count && (now - start_read_time) > @idle_worker_timeout
+              worker.stop
+              worker_stopped = true
+            end
+            end_read_time = now
+          end
+          #Qwirk.logger.debug { "#{self}: start=#{start_read_time} end=#{end_read_time} thres=#{@max_read_threshold} add_new_worker=#{add_new_worker}" }
+          add_new_worker = false if (end_read_time - start_read_time) > @max_read_threshold
+        end
+        add_worker if add_new_worker && @workers.size < @max_count
+      end
+    end
+
+    def to_s
+      @name
+    end
+
+    private
+
+    def add_worker
+      worker = @worker_class.new
+      worker.start(@index_count, self)
+      Qwirk.logger.debug {"#{self}: Adding worker #{worker}"}
+      @index_mutex.synchronize { @index_count += 1 }
+      @workers << worker
+    rescue Exception => e
+      Qwirk.logger.error("Unable to add #{@worker_class} worker: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+    end
+
+    def remove_worker(worker)
+      Qwirk.logger.debug {"#{self}: Deleting worker #{worker}"}
+      @worker_mutex.synchronize do
+        @workers.delete(worker)
+        @worker_condition.broadcast
+      end
+    end
+  end
+end

data/lib/rails/generators/qwirk/qwirk_generator.rb

@@ -0,0 +1,82 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+class QwirkGenerator < Rails::Generators::Base
+  include Rails::Generators::Migration
+
+  def self.source_root
+    File.join(File.dirname(__FILE__), 'templates')
+  end
+   
+  def self.next_migration_number(dirname) #:nodoc:
+    if ActiveRecord::Base.timestamped_migrations
+      Time.now.utc.strftime("%Y%m%d%H%M%S")
+    else
+      "%.3d" % (current_migration_number(dirname) + 1)
+    end
+  end
+
+  def initialize(args, *options) #:nodoc:
+    # Unfreeze name in case it's given as a frozen string
+    args[0] = args[0].dup if args[0].is_a?(String) && args[0].frozen?
+    super
+    assign_names!(self.name)
+    parse_attributes! if respond_to?(:attributes)
+  end
+
+  # Every method that is declared below will be automatically executed when the generator is run
+  
+  def create_migration_file
+    f = File.open File.join(File.dirname(__FILE__), 'templates', 'schema.rb')
+    schema = f.read; f.close
+    
+    schema.gsub!(/ActiveRecord::Schema.*\n/, '')
+    schema.gsub!(/^end\n*$/, '')
+
+    f = File.open File.join(File.dirname(__FILE__), 'templates', 'migration.rb')
+    migration = f.read; f.close
+    migration.gsub!(/SCHEMA_AUTO_INSERTED_HERE/, schema)
+    
+    tmp = File.open "tmp/~migration_ready.rb", "w"
+    tmp.write migration
+    tmp.close
+
+    migration_template  '../../../tmp/~migration_ready.rb',
+                        'db/migrate/create_qwirk_tables.rb'
+    remove_file 'tmp/~migration_ready.rb'
+  end
+
+  def copy_initializer_file
+    copy_file 'initializer.rb', 'config/initializers/qwirk.rb'
+  end
+
+  def update_application_template
+    f = File.open "app/views/layouts/application.html.erb"
+    layout = f.read; f.close
+    
+    if layout =~ /<%=[ ]+yield[ ]+%>/
+      print "    \e[1m\e[34mquestion\e[0m  Your layouts/application.html.erb layout currently has the line <%= yield %>. This gem needs to change this line to <%= content_for?(:content) ? yield(:content) : yield %> to support its nested layouts. This change should not affect any of your existing layouts or views. Is this okay? [y/n] "
+      begin
+        answer = gets.chomp
+      end while not answer =~ /[yn]/i
+      
+      if answer =~ /y/i
+        
+        layout.gsub!(/<%=[ ]+yield[ ]+%>/, '<%= content_for?(:content) ? yield(:content) : yield %>')
+
+        tmp = File.open "tmp/~application.html.erb", "w"
+        tmp.write layout; tmp.close
+
+        remove_file 'app/views/layouts/application.html.erb'
+        copy_file '../../../tmp/~application.html.erb', 
+                  'app/views/layouts/application.html.erb'
+        remove_file 'tmp/~application.html.erb'
+      end
+    elsif layout =~ /<%=[ ]+content_for\?\(:content\) \? yield\(:content\) : yield[ ]+%>/
+      puts "    \e[1m\e[33mskipping\e[0m  layouts/application.html.erb modification is already done."
+    else
+      puts "    \e[1m\e[31mconflict\e[0m  The gem is confused by your layouts/application.html.erb. It does not contain the default line <%= yield %>, you may need to make manual changes to get this gem's nested layouts working. Visit ###### for details."
+    end
+  end
+  
+end