qwirk 0.1.0 → 0.2.0

data/lib/qwirk/adapter/inline/worker.rb

@@ -40,6 +40,10 @@ module Qwirk
         def stop
         end
 
+        def ready_to_stop?
+          true
+        end
+
         ## End of required override methods for worker impl
         private
 

data/lib/qwirk/adapter/inline/worker_config.rb

@@ -16,6 +16,10 @@ module Qwirk
           super.merge(:active => false)
         end
 
+        def self.in_process?(config)
+          true
+        end
+
         # Hack - Steal attribute from expanding_worker_config so test config can share development config
         def min_count=(min_count)
           @active = (min_count > 0)

data/lib/qwirk/adapter_factory.rb

@@ -21,7 +21,7 @@ module Qwirk
       @log_times = config.delete(:log_times)
 
       key = config.delete(:adapter)
-      raise "No adapter definition" unless key
+      raise "No adapter definition config=#{config.inspect}" unless key
       key = key.to_sym
       @publisher_class, @worker_config_class, block = @@adapter_hash[key]
       raise "No adapter matching #{key}" unless @publisher_class
@@ -44,5 +44,9 @@ module Qwirk
     def create_publisher_impl(queue_name, topic_name, options, response_options)
       @publisher_class.new(self, queue_name, topic_name, options, response_options)
     end
+
+    def in_process?
+      @worker_config_class.in_process?(@config)
+    end
   end
 end

data/lib/qwirk/base_worker.rb

@@ -4,7 +4,7 @@ module Qwirk
   # TODO: Is this necessary anymore or just put in worker.rb?  Decide when flat file adapter is implemented.
   module BaseWorker
 
-    attr_accessor :index, :thread, :config
+    attr_accessor :index, :config
 
     module ClassMethods
       def default_name
@@ -71,10 +71,6 @@ module Qwirk
       raise "Need to override stop method in #{self.class.name}"
     end
 
-    def join
-      thread.join
-    end
-
     def status
       raise "Need to override status method in #{self.class.name}"
     end

data/lib/qwirk/engine.rb

@@ -1,9 +1,5 @@
-require 'rails'
-
 module Qwirk
-  class Engine < Rails::Engine
-    initializer "qwirk initialize" , :after =>"active_record.initialize_database" do
-      #config.before_configuration do
-    end
+  class Engine < ::Rails::Engine
+    #isolate_namespace Qwirk
   end
 end

data/lib/qwirk/manager.rb

@@ -9,6 +9,7 @@ module Qwirk
     attr_reader   :env, :worker_configs, :name
 
     bean_attr_accessor :poll_time, :float, 'How often the manager should poll the workers for their status for use by :idle_worker_timeout and :max_read_threshold'
+    bean_attr_accessor :stop_time, :float, 'How long to wait while shutting down for graceful worker stop'
 
     @@default_options = {}
 
@@ -30,7 +31,8 @@ module Qwirk
       options           = @@default_options.merge(options)
       @stopped          = false
       @name             = options[:name] || Qwirk::DEFAULT_NAME
-      @poll_time        = 3.0
+      @poll_time        = options[:poll_time] || 3.0
+      @stop_time        = options[:stop_time]
       @worker_configs   = []
       @env              = options[:env]
       @worker_options   = parse_worker_file(options[:worker_file])
@@ -51,13 +53,12 @@ module Qwirk
       end
 
       start_timer_thread
-      stop_on_signal if options[:stop_on_signal]
+      at_exit { stop }
     end
 
     # Create a timer_thread to make periodic calls to the worker_configs in order to do such things as expand/contract
     # workers, etc.
     def start_timer_thread
-      # TODO: Optionize hard-coded values
       @timer_thread = Thread.new do
         begin
           while !@stopped
@@ -75,17 +76,24 @@ module Qwirk
     def stop
       return if @stopped
       @stopped = true
+      Qwirk.logger.info "Stopping manager"
       @timer_thread.wakeup
       @worker_configs.each { |worker_config| worker_config.stop }
-    end
-
-    def stop_on_signal
-      ['HUP', 'INT', 'TERM'].each do |signal_name|
-        Signal.trap(signal_name) do
-          Qwirk.logger.info "Caught #{signal_name}"
-          stop
+      if @stop_time
+        end_time = Time.now + @stop_time
+        @worker_configs.each do |worker_config|
+          t = end_time - Time.now
+          t = 0 if t < 0
+          worker_config.join(t)
         end
+      else
+        @worker_configs.each { |worker_config| worker_config.join }
       end
+      Qwirk.logger.info "Done stopping manager"
+    end
+
+    def stopped?
+      @stopped
     end
 
     # Store off any options that are no longer set to default

data/lib/qwirk/marshal_strategy/json.rb

@@ -18,7 +18,7 @@ module Qwirk
         end
 
         def unmarshal(msg)
-          ::JSON::Parser.new(msg).parse
+          ::JSON.parse(msg)
         end
 
       rescue LoadError => e

data/lib/qwirk/remote_exception.rb

@@ -30,6 +30,10 @@ module Qwirk
       }
     end
 
+    def marshal
+      to_hash.to_yaml
+    end
+
     def self.from_hash(hash)
       exc = new
       exc.message                       = hash['message']
@@ -38,5 +42,9 @@ module Qwirk
       exc.set_backtrace(hash['backtrace'])
       return exc
     end
+
+    def self.unmarshal(yaml_str)
+      from_hash(YAML.load(yaml_str))
+    end
   end
-end
+end

data/lib/qwirk/task.rb

@@ -1,13 +1,15 @@
 module Qwirk
 
   # The following options can be used for configuring the class
+  #     :max_pending_records => <integer>
+  #       This is how many records can be queued at a time.
   #     :
   module Task
     #include Qwirk::BaseWorker
     include Rumx::Bean
 
+    bean_attr_accessor :max_pending_records, :integer, 'The max number of records that can be published without having been responded to (publishing blocks at this point).'
     bean_attr_reader   :task_id,             :string,  'The ID for this task'
-    bean_attr_reader   :publish_count,       :integer, 'The number of requests that have been published'
     bean_attr_reader   :success_count,       :integer, 'The number of successful responses'
     bean_attr_reader   :exception_count,     :integer, 'The number of exception responses'
     bean_attr_reader   :total_count,         :integer, 'The total expected records to be published (optional)'
@@ -25,21 +27,22 @@ module Qwirk
     end
 
     def initialize(publisher, task_id, total_count, opts={})
-      @publisher           = publisher
-      @mutex               = Mutex.new
-      @condition           = ConditionVariable.new
-      @task_id             = task_id
-      @stopped             = false
-      @finished_publishing = false
-      @retry               = opts[:retry]
-      @auto_retry          = opts[:auto_retry]
-      @publish_count       = 0
-      @success_count       = 0
-      @exception_count     = 0
-      @total_count         = total_count
-      @exceptions_per_run  = []
-
-      @producer, @consumer = publisher.create_producer_consumer_pair(self)
+      @publisher              = publisher
+      @pending_hash           = Hash.new
+      @pending_hash_mutex     = Mutex.new
+      @pending_hash_condition = ConditionVariable.new
+      @task_id                = task_id
+      @stopped                = false
+      @finished_publishing    = false
+      @max_pending_records    = opts[:max_pending_records] || 100
+      @retry                  = opts[:retry]
+      @auto_retry             = opts[:auto_retry]
+      @success_count          = 0
+      @exception_count        = 0
+      @total_count            = total_count
+      @exceptions_per_run     = []
+
+      @producer, @consumer   = publisher.create_producer_consumer_pair(self)
       @reply_thread = Thread.new do
         java.lang.Thread.current_thread.name = "Qwirk task: #{task_id}"
         reply_event_loop
@@ -48,10 +51,10 @@ module Qwirk
     end
 
     # Stuff to override
-    def on_response(response)
+    def on_response(request, response)
     end
 
-    def on_exception(exception)
+    def on_exception(request, exception)
     end
 
     def on_update()
@@ -63,23 +66,19 @@ module Qwirk
     def retry=(val)
       @retry = val
       if val
-        @mutex.synchronize { check_retry }
-      end
-    end
-
-    def auto_retry=(val)
-      @auto_retry = val
-      if val
-        @mutex.synchronize { check_retry }
+        @pending_hash_mutex.synchronize { check_retry }
       end
     end
 
     def publish(object)
       marshaled_object = @publisher.marshaler.marshal(object)
-      @mutex.synchronize do
+      @pending_hash_mutex.synchronize do
+        while !@stopped && @pending_hash.size >= @max_pending_records
+          @pending_hash_condition.wait(@pending_hash_mutex)
+        end
         unless @stopped
-          @producer.send(marshaled_object)
-          @publish_count += 1
+          message_id = @producer.send(marshaled_object)
+          @pending_hash[message_id] = object
         end
       end
     end
@@ -89,14 +88,13 @@ module Qwirk
     end
 
     def stop
-      @mutex.synchronize { do_stop }
+      @pending_hash_mutex.synchronize { do_stop }
       @reply_thread.join
     end
 
     def finished_publishing
-      @total_count = @publish_count
       @finished_publishing = true
-      @mutex.synchronize { check_finish }
+      @pending_hash_mutex.synchronize { check_finish }
       @reply_thread.join
     end
 
@@ -104,6 +102,18 @@ module Qwirk
     private
     #######
 
+    def verify_fail_queue_creation
+      unless @fail_producer
+        @fail_producer, @fail_consumer = publisher.create_producer_fail_consumer_pair(@task_id)
+      end
+    end
+
+    def publish_fail_request(object)
+      verify_fail_queue_creation
+      marshaled_object = @publisher.marshaler.marshal(object)
+      @fail_producer.send(marshaled_object)
+    end
+
     # Must be called within a mutex synchronize
     def do_stop
       return if @stopped
@@ -113,19 +123,26 @@ module Qwirk
     end
 
     def reply_event_loop
-      while !@stopped && response = @consumer.receive
-        @mutex.synchronize do
+      while !@stopped && pair = @consumer.receive
+        message_id, response = pair
+        @pending_hash_mutex.synchronize do
           unless @stopped
-            if response.kind_of?(RemoteException)
-              @exception_count += 1
-              on_exception(response)
+            request = @pending_hash.delete(message_id)
+            if request
+              if response.kind_of?(RemoteException)
+                publish_fail_request(request)
+                on_exception(request, response)
+                @exception_count += 1
+              else
+                on_response(request, response)
+                @success_count += 1
+              end
             else
-              @success_count += 1
-              on_response(response)
+              Qwirk.logger.warn("#{self}: Read unexpected response with message_id=#{message_id}")
             end
             @consumer.acknowledge_message
             check_finish
-            @condition.signal
+            @pending_hash_condition.signal
           end
         end
       end
@@ -138,8 +155,8 @@ module Qwirk
 
     # Must be called within a mutex synchronize
     def check_finish
-      if @finished_publishing
-        if @success_count >= @total_count
+      if @finished_publishing && @pending_hash.empty?
+        if @exception_count == 0
           do_stop
         else
           check_retry
@@ -149,7 +166,7 @@ module Qwirk
 
     # Must be called within a mutex synchronize
     def check_retry
-      if @finished_publishing && @exception_count > 0 && (@exception_count+@success_count) == @total_count && (@retry || @auto_retry)
+      if @finished_publishing && @pending_hash.empty? && @exception_count > 0 && (@retry || @auto_retry)
         # If we're just doing auto_retry but nothing succeeded last time, then don't run again
         return if !@retry && @auto_retry && @exception_count == @exceptions_per_run.last
         Qwirk.logger.info "#{self}: Retrying exception records, exception count = #{@exception_count}"
@@ -165,7 +182,7 @@ module Qwirk
               @fail_consumer.acknowledge_message
             end
             @finished_publishing = true
-            @mutex.synchronize { check_finish }
+            @pending_hash_mutex.synchronize { check_finish }
           rescue Exception => e
             do_stop
             Qwirk.logger.error "#{self}: Exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"

data/lib/qwirk/worker.rb

@@ -37,7 +37,7 @@ module Qwirk
     include Qwirk::BaseWorker
 
     attr_accessor :message
-    attr_reader   :status, :impl, :start_worker_time, :start_read_time, :start_processing_time
+    attr_reader   :thread, :status, :impl, :start_worker_time, :start_read_time, :start_processing_time
 
     module ClassMethods
       def queue(name, opts={})
@@ -122,7 +122,7 @@ module Qwirk
     end
 
     def start
-      self.thread = Thread.new do
+      @thread = Thread.new do
         java.lang.Thread.current_thread.name = "Qwirk worker: #{self}" if RUBY_PLATFORM == 'jruby'
         #Qwirk.logger.debug "#{worker}: Started thread with priority #{Thread.current.priority}"
         event_loop
@@ -136,14 +136,25 @@ module Qwirk
     # From an InMemory perspective, we don't want the workers stopping until all messages in the queue have been processed.
     # Therefore we want to stop the
     def stop
-      Qwirk.logger.debug "#{self}: In base worker stop"
+      Qwirk.logger.debug "#{self}: In worker stop"
       @status  = 'Stopping'
       @stopped = true
       @processing_mutex.synchronize do
         # This should interrupt @impl.receive_message above and cause it to return nil
         @impl.stop
       end
-      Qwirk.logger.debug "#{self}: base worker stop complete"
+    end
+
+    def join(timeout=nil)
+      time_str = timeout ? "#{timeout} seconds" : 'indefinitely'
+      Qwirk.logger.info "#{self}: Waiting #{time_str} for worker to stop"
+      # See http://jira.codehaus.org/browse/JRUBY-6896
+      if timeout && !self.thread.join(timeout)
+        Qwirk.logger.warn "#{self}: WARNING - worker stop commanded but thread did not exit gracefully in #{timeout} seconds"
+      else
+        self.thread.join
+      end
+      Qwirk.logger.info "#{self}: worker stop complete"
     end
 
     def perform(object)
@@ -163,7 +174,7 @@ module Qwirk
     def event_loop
       Qwirk.logger.debug "#{self}: Starting receive loop"
       @start_worker_time = Time.now
-      while !@stopped && !config.stopped
+      until @stopped || (config.stopped? && @impl.ready_to_stop?)
         Qwirk.logger.debug "#{self}: Waiting for read"
         @start_read_time = Time.now
         msg = @impl.receive_message
@@ -186,8 +197,6 @@ module Qwirk
       Qwirk.logger.error "#{self}: Exception, thread terminating: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
     ensure
       @status = 'Stopped'
-      # TODO: necessary?
-      @impl.stop
       Qwirk.logger.flush if Qwirk.logger.respond_to?(:flush)
       config.worker_stopped(self)
     end

data/test/dummy/README.rdoc

@@ -0,0 +1,261 @@
+== Welcome to Rails
+
+Rails is a web-application framework that includes everything needed to create
+database-backed web applications according to the Model-View-Control pattern.
+
+This pattern splits the view (also called the presentation) into "dumb"
+templates that are primarily responsible for inserting pre-built data in between
+HTML tags. The model contains the "smart" domain objects (such as Account,
+Product, Person, Post) that holds all the business logic and knows how to
+persist themselves to a database. The controller handles the incoming requests
+(such as Save New Account, Update Product, Show Post) by manipulating the model
+and directing data to the view.
+
+In Rails, the model is handled by what's called an object-relational mapping
+layer entitled Active Record. This layer allows you to present the data from
+database rows as objects and embellish these data objects with business logic
+methods. You can read more about Active Record in
+link:files/vendor/rails/activerecord/README.html.
+
+The controller and view are handled by the Action Pack, which handles both
+layers by its two parts: Action View and Action Controller. These two layers
+are bundled in a single package due to their heavy interdependence. This is
+unlike the relationship between the Active Record and Action Pack that is much
+more separate. Each of these packages can be used independently outside of
+Rails. You can read more about Action Pack in
+link:files/vendor/rails/actionpack/README.html.
+
+
+== Getting Started
+
+1. At the command prompt, create a new Rails application:
+       <tt>rails new myapp</tt> (where <tt>myapp</tt> is the application name)
+
+2. Change directory to <tt>myapp</tt> and start the web server:
+       <tt>cd myapp; rails server</tt> (run with --help for options)
+
+3. Go to http://localhost:3000/ and you'll see:
+       "Welcome aboard: You're riding Ruby on Rails!"
+
+4. Follow the guidelines to start developing your application. You can find
+the following resources handy:
+
+* The Getting Started Guide: http://guides.rubyonrails.org/getting_started.html
+* Ruby on Rails Tutorial Book: http://www.railstutorial.org/
+
+
+== Debugging Rails
+
+Sometimes your application goes wrong. Fortunately there are a lot of tools that
+will help you debug it and get it back on the rails.
+
+First area to check is the application log files. Have "tail -f" commands
+running on the server.log and development.log. Rails will automatically display
+debugging and runtime information to these files. Debugging info will also be
+shown in the browser on requests from 127.0.0.1.
+
+You can also log your own messages directly into the log file from your code
+using the Ruby logger class from inside your controllers. Example:
+
+  class WeblogController < ActionController::Base
+    def destroy
+      @weblog = Weblog.find(params[:id])
+      @weblog.destroy
+      logger.info("#{Time.now} Destroyed Weblog ID ##{@weblog.id}!")
+    end
+  end
+
+The result will be a message in your log file along the lines of:
+
+  Mon Oct 08 14:22:29 +1000 2007 Destroyed Weblog ID #1!
+
+More information on how to use the logger is at http://www.ruby-doc.org/core/
+
+Also, Ruby documentation can be found at http://www.ruby-lang.org/. There are
+several books available online as well:
+
+* Programming Ruby: http://www.ruby-doc.org/docs/ProgrammingRuby/ (Pickaxe)
+* Learn to Program: http://pine.fm/LearnToProgram/ (a beginners guide)
+
+These two books will bring you up to speed on the Ruby language and also on
+programming in general.
+
+
+== Debugger
+
+Debugger support is available through the debugger command when you start your
+Mongrel or WEBrick server with --debugger. This means that you can break out of
+execution at any point in the code, investigate and change the model, and then,
+resume execution! You need to install ruby-debug to run the server in debugging
+mode. With gems, use <tt>sudo gem install ruby-debug</tt>. Example:
+
+  class WeblogController < ActionController::Base
+    def index
+      @posts = Post.all
+      debugger
+    end
+  end
+
+So the controller will accept the action, run the first line, then present you
+with a IRB prompt in the server window. Here you can do things like:
+
+  >> @posts.inspect
+  => "[#<Post:0x14a6be8
+          @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>,
+       #<Post:0x14a6620
+          @attributes={"title"=>"Rails", "body"=>"Only ten..", "id"=>"2"}>]"
+  >> @posts.first.title = "hello from a debugger"
+  => "hello from a debugger"
+
+...and even better, you can examine how your runtime objects actually work:
+
+  >> f = @posts.first
+  => #<Post:0x13630c4 @attributes={"title"=>nil, "body"=>nil, "id"=>"1"}>
+  >> f.
+  Display all 152 possibilities? (y or n)
+
+Finally, when you're ready to resume execution, you can enter "cont".
+
+
+== Console
+
+The console is a Ruby shell, which allows you to interact with your
+application's domain model. Here you'll have all parts of the application
+configured, just like it is when the application is running. You can inspect
+domain models, change values, and save to the database. Starting the script
+without arguments will launch it in the development environment.
+
+To start the console, run <tt>rails console</tt> from the application
+directory.
+
+Options:
+
+* Passing the <tt>-s, --sandbox</tt> argument will rollback any modifications
+  made to the database.
+* Passing an environment name as an argument will load the corresponding
+  environment. Example: <tt>rails console production</tt>.
+
+To reload your controllers and models after launching the console run
+<tt>reload!</tt>
+
+More information about irb can be found at:
+link:http://www.rubycentral.org/pickaxe/irb.html
+
+
+== dbconsole
+
+You can go to the command line of your database directly through <tt>rails
+dbconsole</tt>. You would be connected to the database with the credentials
+defined in database.yml. Starting the script without arguments will connect you
+to the development database. Passing an argument will connect you to a different
+database, like <tt>rails dbconsole production</tt>. Currently works for MySQL,
+PostgreSQL and SQLite 3.
+
+== Description of Contents
+
+The default directory structure of a generated Ruby on Rails application:
+
+  |-- app
+  |   |-- assets
+  |       |-- images
+  |       |-- javascripts
+  |       `-- stylesheets
+  |   |-- controllers
+  |   |-- helpers
+  |   |-- mailers
+  |   |-- models
+  |   `-- views
+  |       `-- layouts
+  |-- config
+  |   |-- environments
+  |   |-- initializers
+  |   `-- locales
+  |-- db
+  |-- doc
+  |-- lib
+  |   `-- tasks
+  |-- log
+  |-- public
+  |-- script
+  |-- test
+  |   |-- fixtures
+  |   |-- functional
+  |   |-- integration
+  |   |-- performance
+  |   `-- unit
+  |-- tmp
+  |   |-- cache
+  |   |-- pids
+  |   |-- sessions
+  |   `-- sockets
+  `-- vendor
+      |-- assets
+          `-- stylesheets
+      `-- plugins
+
+app
+  Holds all the code that's specific to this particular application.
+
+app/assets
+  Contains subdirectories for images, stylesheets, and JavaScript files.
+
+app/controllers
+  Holds controllers that should be named like weblogs_controller.rb for
+  automated URL mapping. All controllers should descend from
+  ApplicationController which itself descends from ActionController::Base.
+
+app/models
+  Holds models that should be named like post.rb. Models descend from
+  ActiveRecord::Base by default.
+
+app/views
+  Holds the template files for the view that should be named like
+  weblogs/index.html.erb for the WeblogsController#index action. All views use
+  eRuby syntax by default.
+
+app/views/layouts
+  Holds the template files for layouts to be used with views. This models the
+  common header/footer method of wrapping views. In your views, define a layout
+  using the <tt>layout :default</tt> and create a file named default.html.erb.
+  Inside default.html.erb, call <% yield %> to render the view using this
+  layout.
+
+app/helpers
+  Holds view helpers that should be named like weblogs_helper.rb. These are
+  generated for you automatically when using generators for controllers.
+  Helpers can be used to wrap functionality for your views into methods.
+
+config
+  Configuration files for the Rails environment, the routing map, the database,
+  and other dependencies.
+
+db
+  Contains the database schema in schema.rb. db/migrate contains all the
+  sequence of Migrations for your schema.
+
+doc
+  This directory is where your application documentation will be stored when
+  generated using <tt>rake doc:app</tt>
+
+lib
+  Application specific libraries. Basically, any kind of custom code that
+  doesn't belong under controllers, models, or helpers. This directory is in
+  the load path.
+
+public
+  The directory available for the web server. Also contains the dispatchers and the
+  default HTML files. This should be set as the DOCUMENT_ROOT of your web
+  server.
+
+script
+  Helper scripts for automation and generation.
+
+test
+  Unit and functional tests along with fixtures. When using the rails generate
+  command, template test files will be generated for you and placed in this
+  directory.
+
+vendor
+  External libraries that the application depends on. Also includes the plugins
+  subdirectory. If the app has frozen rails, those gems also go here, under
+  vendor/rails/. This directory is in the load path.