derfred-workling 0.4.9.2 → 0.4.9.3

data/lib/workling/remote/invokers/basic_poller.rb

@@ -1,41 +0,0 @@
-require 'workling/remote/invokers/base'
-
-#
-#  A basic polling invoker. 
-#  
-module Workling
-  module Remote
-    module Invokers
-      class BasicPoller < Workling::Remote::Invokers::Base
-        
-        #
-        #  set up client, sleep time
-        #
-        def initialize(routing, client_class)
-          super
-        end
-        
-        #
-        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
-        #
-        def listen
-          connect do
-            loop_routes do |route|
-              if args = @client.retrieve(route)
-                run(route, args)
-              end
-            end
-          end
-        end
-        
-        #
-        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
-        #  to finish. 
-        #
-        def stop
-          @shutdown = true
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/eventmachine_subscriber.rb

@@ -1,41 +0,0 @@
-require 'eventmachine'
-require 'workling/remote/invokers/base'
-
-#
-#  Subscribes the workers to the correct queues. 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class EventmachineSubscriber < Workling::Remote::Invokers::Base
-        
-        def initialize(routing, client_class)
-          super
-        end
-        
-        #
-        #  Starts EM loop and sets up subscription callbacks for workers. 
-        #
-        def listen
-          EM.run do
-            connect do
-              routes.each do |route|
-                @client.subscribe(route) do |args|
-                  begin
-                    run(route, args)
-                  rescue
-                    logger.error("EventmachineSubscriber listen error on #{route}: #{$!}")
-                  end
-                end
-              end
-            end
-          end
-        end
-                
-        def stop
-          EM.stop if EM.reactor_running?
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/looped_subscriber.rb

@@ -1,38 +0,0 @@
-require 'workling/remote/invokers/base'
-
-#
-#  Subscribes the workers to the correct queues. 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class LoopedSubscriber < Workling::Remote::Invokers::Base
-
-        def initialize(routing, client_class)
-          super
-        end
-
-        #
-        #  Starts EM loop and sets up subscription callbacks for workers. 
-        #
-        def listen
-          connect do
-            routes.each do |route|
-              @client.subscribe(route) do |args|
-                run(route, args)
-              end
-            end
-
-            loop do
-              sleep 1
-            end
-          end
-        end
-
-        def stop
-          
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/thread_pool_poller.rb

@@ -1,169 +0,0 @@
-require 'workling/remote/invokers/base'
-
-require 'thread'
-require 'mutex_m'
-
-#
-# A polling invoker that executes the jobs using a thread pool.
-#
-# This invoker was designed for long running tasks and Rails 2.2. It is expected
-# that each worker will manage it's own database connections using ActiveRecord::Base.connection_pool.
-#
-# This implementation isn't using most of the features provided by Invokers::Base
-# because of it's assumptions about database connections
-#
-# Example:
-#   user = nil
-#   ActiveRecord::Base.connection_pool.with_connection do
-#     user = User.find(options[:id])
-#   end
-#
-#   # Do long running stuff here
-#   user.wait_until_birthday
-#   user.buy_gift 'BRAWNDO!'
-#
-#   ActiveRecord::Base.connection_pool.with_connection do
-#     user.save
-#   end
-#
-module Workling
-  module Remote
-    module Invokers
-      class ThreadPoolPoller < Workling::Remote::Invokers::Base
-        attr_reader :sleep_time, :reset_time, :pool_capacity
-
-        def initialize(routing, client_class)
-          @routing = routing
-          @client_class = client_class
-
-          # Grab settings out of the config file
-          @sleep_time = (Workling.config[:sleep_time] || 2).to_f
-          @reset_time = (Workling.config[:reset_time] || 30).to_f
-
-          # Pool of polling threads
-          @pollers = []
-          @pollers.extend(Mutex_m)
-
-          # Pool of worker threads
-          @workers = []
-          @workers.extend(Mutex_m)
-
-          # Connection to the job queue
-          @pool_capacity = (Workling.config[:pool_size] || 25).to_i
-        end
-
-        # Start up the checking for items on the queue. Will block until stop is called and all pollers
-        # and workers have finished execution.
-        def listen
-          logger.info("Starting ThreadPoolPoller...")
-
-          # Determine which queues need checking
-          Workling::Discovery.discovered.map do |klass|
-            @pollers.synchronize do
-              # Polls the backing queue for jobs to be done
-              @pollers << Thread.new do
-                poller_thread(@routing.queue_names_routing_class(klass))
-              end
-            end
-          end
-
-          # Wait for the poller and all outstanding workers to finish.
-          #
-          # This is a little tricky because we're doing some synchronization on pollers... but
-          # the list of pollers is never modified after being setup above.
-          @pollers.synchronize { @pollers.dup }.each { |p| p.join }
-          @pollers.synchronize { @pollers.clear }
-          logger.info("Pollers have all finished")
-
-          @workers.synchronize { @workers.dup }.each { |w| w.join }
-          logger.info("Worker threads have all finished")
-        end
-
-        # Instructs the thread pool poller to stop checking for new jobs on the backing queue.
-        def stop
-          logger.info("Stopping thread pool invoker pollers and workers...")
-          @pollers.synchronize { @pollers.each { |p| p[:shutdown] = true } }
-        end
-
-        # Set pool_size in workling config to adjust the maximum number of threads in the pool
-        def workers_available?
-          worker_threads < @pool_capacity
-        end
-
-        # Number of correctly active worker threads
-        def worker_threads
-          @workers.synchronize { @workers.size }
-        end
-
-        # Number of currently active polling threads
-        def poller_threads
-          @pollers.synchronize { @pollers.size }
-        end
-
-        private
-
-        def poller_thread(queues)
-          # Make sure queues is an array
-          queues = [queues].flatten!
-
-          # Connect our client to the backing queue
-          client = @client_class.new
-          client.connect
-          logger.info("** Starting client #{ client.class } for #{ queues.inspect }") if logger.info?
-
-          # Poll each queue for new items
-          while(!Thread.current[:shutdown]) do
-            # Check each queue for a job posting
-            queues.each do |queue|
-              break if Thread.current[:shutdown]
-
-              begin
-                # Take a job off the queue and execute it in a new worker thread.
-                #
-                # Don't pop any jobs off the backing queue if the thread pool
-                # is full. This keeps them on the master queue so other instances
-                # can process them.
-                while(workers_available? && (options = client.retrieve(queue)))
-                  logger.debug("#{queue} received job #{ options.inspect }") if logger.debug?
-
-                  @workers.synchronize do
-                    @workers << Thread.new do
-                      begin
-                        # Execute the job
-                        run(queue, options)
-                      rescue Exception => e
-                        # Log the exception since there isn't much else we can do about it at this point
-                        logger.error(e) if logger.error?
-                      ensure
-                        # Make sure the current thread's connection gets released
-                        if(ActiveRecord::Base.connection_pool)
-                          ActiveRecord::Base.connection_pool.release_connection
-                        end
-
-                        # Remove this thread from the list of active workers
-                        @workers.synchronize do
-                          @workers.delete(Thread.current)
-                        end
-                      end
-                    end
-                  end
-
-                  # Break out of the checks early if shutdown was called
-                  break if Thread.current[:shutdown]
-                end
-              rescue Workling::WorklingError => e
-                logger.error("FAILED to connect with queue #{ queue }: #{ e } }") if logger.error?
-                sleep(@reset_time)
-
-                # FIXME: This will _definitely_ blow up with AMQP since there is no reset call
-                client.reset
-              end
-            end
-
-            sleep(@sleep_time) unless Thread.current[:shutdown]
-          end
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/threaded_poller.rb

@@ -1,153 +0,0 @@
-require 'workling/remote/invokers/base'
-
-#
-#  A threaded polling Invoker. 
-# 
-#  TODO: refactor this to make use of the base class. 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class ThreadedPoller < Workling::Remote::Invokers::Base
-        
-        cattr_accessor :sleep_time, :reset_time
-      
-        def initialize(routing, client_class)
-          super
-          
-          ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
-          ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
-          
-          @workers = ThreadGroup.new
-          @mutex = Mutex.new
-        end      
-          
-        def listen
-          # Create a thread for each worker.
-          Workling::Discovery.discovered.each do |clazz|
-            logger.debug("Discovered listener #{clazz}")
-            @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
-          end
-          
-          # Wait for all workers to complete
-          @workers.list.each { |t| t.join }
-
-          logger.debug("Reaped listener threads. ")
-        
-          # Clean up all the connections.
-          if defined?(ActiveRecord::Base)
-            ActiveRecord::Base.verify_active_connections!
-          end
-
-          logger.debug("Cleaned up connection: out!")
-        end
-      
-        # Check if all Worker threads have been started. 
-        def started?
-          logger.debug("checking if started... list size is #{ worker_threads }")
-          Workling::Discovery.discovered.size == worker_threads
-        end
-        
-        # number of worker threads running
-        def worker_threads
-          @workers.list.size
-        end
-      
-        # Gracefully stop processing
-        def stop
-          logger.info("stopping threaded poller...")
-          sleep 1 until started? # give it a chance to start up before shutting down. 
-          logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
-          @workers.list.each { |w| w[:shutdown] = true }
-          logger.info("Listener threads were shut down.  ")
-        end
-
-        # Listen for one worker class
-        def clazz_listen(clazz)
-          logger.debug("Listener thread #{clazz.name} started")
-           
-          # Read thread configuration if available
-          if Workling.config.has_key?(:listeners)
-            if Workling.config[:listeners].has_key?(clazz.to_s)
-              config = Workling.config[:listeners][clazz.to_s].symbolize_keys
-              thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
-              Thread.current.priority = config[:priority] if config.has_key?(:priority)
-            end
-          end
-
-          thread_sleep_time ||= self.class.sleep_time
-                
-          # Setup connection to client (one per thread)
-          connection = @client_class.new
-          connection.connect
-          logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
-     
-          # Start dispatching those messages
-          while (!Thread.current[:shutdown]) do
-            begin
-            
-              # Thanks for this Brent! 
-              #
-              #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
-              #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
-              #     to wrap the code that checks for a connection in in a mutex.
-              #
-              #     ....I noticed this while working with a multi-core machine that 
-              #     was spawning multiple workling threads. Some of my workling 
-              #     threads would hit serious issues at this block of code without 
-              #     the mutex.            
-              #
-              if defined?(ActiveRecord::Base)
-                @mutex.synchronize do 
-                  unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
-                    unless ActiveRecord::Base.connection.reconnect!
-                      logger.fatal("Failed - Database not available!")
-                      break
-                    end
-                  end
-                end
-              end
-
-              # Dispatch and process the messages
-              n = dispatch!(connection, clazz)
-              logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
-              sleep(thread_sleep_time) unless n > 0
-            
-            # If there is a memcache error, hang for a bit to give it a chance to fire up again
-            # and reset the connection.
-            rescue Workling::WorklingConnectionError
-              logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
-              sleep(self.class.reset_time)
-              connection.reset
-            end
-          end
-        
-          logger.debug("Listener thread #{clazz.name} ended")
-        end
-      
-        # Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
-        # Returns the number of worker methods called
-        def dispatch!(connection, clazz)
-          n = 0
-          for queue in @routing.queue_names_routing_class(clazz)
-            begin
-              result = connection.retrieve(queue)
-              if result
-                n += 1
-                handler = @routing[queue]
-                method_name = @routing.method_name(queue)
-                logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
-                handler.dispatch_to_worker_method(method_name, result)
-              end
-            rescue Workling::WorklingError => e
-              logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
-              raise e
-            end
-          end
-        
-          return n
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/amqp_exchange_runner.rb

@@ -1,45 +0,0 @@
-require 'workling/remote/runners/base'
-
-#
-#  Runs Jobs over a Client. The client should be a subclass of Workling::Client::Base. 
-#  Set the client like this: 
-#
-#      Workling::Remote::Runners::ClientRunner.client = Workling::Clients::AmqpClient.new
-#
-#  Jobs are dispatched by requesting them on the Client. The Runner takes care of mapping of queue names to worker code. 
-#  this is done with Workling::ClassAndMethodRouting, but you can use your own by sublassing Workling::Routing. 
-#  Don’t worry about any of this if you’re not dealing directly with the queues.
-#
-#  There’s a workling-client daemon that uses the configured invoker to retrieve work and dispatching these to the 
-#  responsible workers. If you intend to run this on a remote machine, then just check out your rails project 
-#  there and start up the workling client like this: ruby script/workling_client run.
-#
-module Workling
-  module Remote
-    module Runners
-      class AmqpExchangeRunner < Workling::Remote::Runners::Base
-        
-        # Routing class. Workling::Routing::ClassAndMethodRouting.new by default. 
-        cattr_accessor :routing
-        @@routing ||= Workling::Routing::ClassAndMethodRouting.new
-        
-        # The workling Client class. Workling::Clients::MemcacheQueueClient.new by default. 
-        cattr_accessor :client
-        @@client ||= Workling::Clients::AmqpExchangeClient.new
-        
-        # enqueues the job onto the client
-        def run(clazz, method, options = {})
-          
-          # neet to connect in here as opposed to the constructor, since the EM loop is
-          # not available there. 
-          @connected ||= self.class.client.connect
-
-          # NOTE - currently hardcoded to use the default exchange
-          self.class.client.request("amq.topic", options)    
-          
-          return nil
-        end
-      end
-    end
-  end
-end