elecnix-workling 0.4.2 → 0.4.9

data/lib/workling/remote/invokers/basic_poller.rb

@@ -1,41 +0,0 @@
-require 'workling/remote/invokers/base'
-
-#
-#  A basic polling invoker. 
-#  
-module Workling
-  module Remote
-    module Invokers
-      class BasicPoller < Workling::Remote::Invokers::Base
-        
-        #
-        #  set up client, sleep time
-        #
-        def initialize(routing, client_class)
-          super
-        end
-        
-        #
-        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
-        #
-        def listen
-          connect do
-            loop_routes do |route|
-              if args = @client.retrieve(route)
-                run(route, args)
-              end
-            end
-          end
-        end
-        
-        #
-        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
-        #  to finish. 
-        #
-        def stop
-          @shutdown = true
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/eventmachine_subscriber.rb

@@ -1,41 +0,0 @@
-require 'eventmachine'
-require 'workling/remote/invokers/base'
-
-#
-#  Subscribes the workers to the correct queues. 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class EventmachineSubscriber < Workling::Remote::Invokers::Base
-        
-        def initialize(routing, client_class)
-          super
-        end
-        
-        #
-        #  Starts EM loop and sets up subscription callbacks for workers. 
-        #
-        def listen
-          EM.run do
-            connect do
-              routes.each do |route|
-                @client.subscribe(route) do |args|
-                  begin
-                    run(route, args)
-                  rescue
-                    logger.error("EventmachineSubscriber listen error on #{route}: #{$!}")
-                  end
-                end
-              end
-            end
-          end
-        end
-                
-        def stop
-          EM.stop if EM.reactor_running?
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/threaded_poller.rb

@@ -1,140 +0,0 @@
-require 'workling/remote/invokers/base'
-
-#
-#  A threaded polling Invoker. 
-# 
-#  TODO: refactor this to make use of the base class. 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class ThreadedPoller < Workling::Remote::Invokers::Base
-        
-        cattr_accessor :sleep_time, :reset_time
-      
-        def initialize(routing, client_class)
-          super
-          
-          ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
-          ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
-          
-          @workers = ThreadGroup.new
-          @mutex = Mutex.new
-        end      
-          
-        def listen
-          # Create a thread for each worker.
-          Workling::Discovery.discovered.each do |clazz|
-            logger.debug("Discovered listener #{clazz}")
-            @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
-          end
-          
-          # Wait for all workers to complete
-          @workers.list.each { |t| t.join }
-
-          logger.debug("Reaped listener threads. ")
-        
-          # Clean up all the connections.
-          ActiveRecord::Base.verify_active_connections!
-          logger.debug("Cleaned up connection: out!")
-        end
-      
-        # Check if all Worker threads have been started. 
-        def started?
-          logger.debug("checking if started... list size is #{ worker_threads }")
-          Workling::Discovery.discovered.size == worker_threads
-        end
-        
-        # number of worker threads running
-        def worker_threads
-          @workers.list.size
-        end
-      
-        # Gracefully stop processing
-        def stop
-          logger.info("stopping threaded poller...")
-          sleep 1 until started? # give it a chance to start up before shutting down. 
-          logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
-          @workers.list.each { |w| w[:shutdown] = true }
-          logger.info("Listener threads were shut down.  ")
-        end
-
-        # Listen for one worker class
-        def clazz_listen(clazz)
-          logger.debug("Listener thread #{clazz.name} started")
-           
-          # Read thread configuration if available
-          if Workling.config.has_key?(:listeners)
-            if Workling.config[:listeners].has_key?(clazz.to_s)
-              config = Workling.config[:listeners][clazz.to_s].symbolize_keys
-              thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
-              Thread.current.priority = config[:priority] if config.has_key?(:priority)
-            end
-          end
-
-          thread_sleep_time ||= self.class.sleep_time
-                
-          # Setup connection to client (one per thread)
-          connection = @client_class.new
-          connection.connect
-          logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
-     
-          # Start dispatching those messages
-          while (!Thread.current[:shutdown]) do
-            begin
-            
-              # Wrap code calling ActiveRecord::Base.connection_active? in a
-              # mutex to avoid problem with spawning threads on multi-core
-              # machines running MySQL.
-              @mutex.synchronize do 
-                unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
-                  unless ActiveRecord::Base.connection.reconnect!
-                    logger.fatal("Failed - Database not available!")
-                    break
-                  end
-                end
-              end
-
-              # Dispatch and process the messages
-              n = dispatch!(connection, clazz)
-              logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
-              sleep(thread_sleep_time) unless n > 0
-            
-              # If there is a memcache error, hang for a bit to give it a chance to fire up again
-              # and reset the connection.
-              rescue Workling::WorklingConnectionError
-                logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
-                sleep(self.class.reset_time)
-                connection.reset
-            end
-          end
-        
-          logger.debug("Listener thread #{clazz.name} ended")
-        end
-      
-        # Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
-        # Returns the number of worker methods called
-        def dispatch!(connection, clazz)
-          n = 0
-          for queue in @routing.queue_names_routing_class(clazz)
-            begin
-              result = connection.retrieve(queue)
-              if result
-                n += 1
-                handler = @routing[queue]
-                method_name = @routing.method_name(queue)
-                logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
-                handler.dispatch_to_worker_method(method_name, result)
-              end
-            rescue MemCache::MemCacheError => e
-              logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
-              raise e
-            end
-          end
-        
-          return n
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/backgroundjob_runner.rb

@@ -1,35 +0,0 @@
-require 'workling/remote/runners/base'
-
-#
-#  Use Ara Howards BackgroundJob to run the work. BackgroundJob loads Rails once per requested Job. 
-#  It persists over the database, and there is no requirement for separate processes to be started. 
-#  Since rails has to load before each request, it takes a moment for the job to run. 
-#
-module Workling
-  module Remote
-    module Runners
-      class BackgroundjobRunner < Workling::Remote::Runners::Base
-        cattr_accessor :routing
-        
-        def initialize
-          BackgroundjobRunner.routing = 
-            Workling::Routing::ClassAndMethodRouting.new
-        end
-        
-        #  passes the job to bj by serializing the options to xml and passing them to
-        #  ./script/bj_invoker.rb, which in turn routes the deserialized args to the
-        #  appropriate worker. 
-        def run(clazz, method, options = {})
-          stdin = @@routing.queue_for(clazz, method) + 
-                  " " + 
-                  options.to_xml(:indent => 0, :skip_instruct => true)
-                  
-          Bj.submit "./script/runner ./script/bj_invoker.rb", 
-            :stdin => stdin
-          
-          return nil # that means nothing!
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/base.rb

@@ -1,42 +0,0 @@
-#
-#  Base class for Workling Runners.
-#
-#  Runners must subclass this and implement the method 
-#
-#     Workling::Remote::Runners::Base#run(clazz, method, options = {})
-#
-#  which is responsible for pushing the requested job into the background. Depending
-#  on the Runner, this may require other code to dequeue the job. The actual
-#  invocation of the runner should be done like this: 
-#
-#      Workling.find(clazz, method).dispatch_to_worker_method(method, options)
-#
-#  This ensures for consistent logging and handling of propagated exceptions. You can
-#  also call the convenience method 
-#       
-#      Workling::Remote::Runners::Base#dispatch!(clazz, method, options)
-#
-#  which invokes this for you. 
-#
-module Workling
-  module Remote
-    module Runners
-      class Base
-        
-        # runner uses this to connect to a job broker
-        cattr_accessor :client
-        
-        # default logger defined in Workling::Base.logger
-        def logger
-          Workling::Base.logger
-        end
-
-        # find the worker instance and invoke it. Invoking the worker method like this ensures for 
-        # consistent logging and handling of propagated exceptions. 
-        def dispatch!(clazz, method, options)
-          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/client_runner.rb

@@ -1,45 +0,0 @@
-require 'workling/remote/runners/base'
-require 'workling/clients/memcache_queue_client'
-
-#
-#  Runs Jobs over a Client. The client should be a subclass of Workling::Client::Base. 
-#  Set the client like this: 
-#
-#      Workling::Remote::Runners::ClientRunner.client = Workling::Clients::AmqpClient.new
-#
-#  Jobs are dispatched by requesting them on the Client. The Runner takes care of mapping of queue names to worker code. 
-#  this is done with Workling::ClassAndMethodRouting, but you can use your own by sublassing Workling::Routing. 
-#  Don’t worry about any of this if you’re not dealing directly with the queues.
-#
-#  There’s a workling-client daemon that uses the configured invoker to retrieve work and dispatching these to the 
-#  responsible workers. If you intend to run this on a remote machine, then just check out your rails project 
-#  there and start up the workling client like this: ruby script/workling_client run.
-#
-module Workling
-  module Remote
-    module Runners
-      class ClientRunner < Workling::Remote::Runners::Base
-        
-        # Routing class. Workling::Routing::ClassAndMethodRouting.new by default. 
-        cattr_accessor :routing
-        @@routing ||= Workling::Routing::ClassAndMethodRouting.new
-        
-        # The workling Client class. Workling::Clients::MemcacheQueueClient.new by default. 
-        cattr_accessor :client
-        @@client ||= Workling::Clients::MemcacheQueueClient.new
-        
-        # enqueues the job onto the client
-        def run(clazz, method, options = {})
-          
-          # neet to connect in here as opposed to the constructor, since the EM loop is
-          # not available there. 
-          @connected ||= self.class.client.connect
-          
-          self.class.client.request(@@routing.queue_for(clazz, method), options)    
-          
-          return nil
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/not_remote_runner.rb

@@ -1,23 +0,0 @@
-require 'workling/remote/runners/base'
-
-#
-# directly dispatches to the worker method, in-process. options are first marshalled then dumped
-# in order to simulate the sideeffects of a remote call.
-#
-module Workling
-  module Remote
-    module Runners
-      class NotRemoteRunner < Workling::Remote::Runners::Base
-        
-        # directly dispatches to the worker method, in-process. options are first marshalled then dumped
-        # in order to simulate the sideeffects of a remote call. 
-        def run(clazz, method, options = {})
-          options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
-          dispatch!(clazz, method, options) 
-          
-          return nil # nada. niente.
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/rudeq_runner.rb

@@ -1,23 +0,0 @@
-require 'workling/remote/runners/base'
-
-module Workling
-  module Remote
-    module Runners
-      class RudeqRunner < Workling::Remote::Runners::Base
-        cattr_accessor :routing
-        cattr_accessor :client
-        
-        def initialize
-          RudeqRunner.client = Workling::Rudeq::Client.new
-          RudeqRunner.routing = Workling::Starling::Routing::ClassAndMethodRouting.new
-        end
-        
-        def run(clazz, method, options = {})
-          RudeqRunner.client.set(@@routing.queue_for(clazz, method), options)
-          
-          return nil # empty.
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/spawn_runner.rb

@@ -1,38 +0,0 @@
-require 'workling/remote/runners/base'
-
-#
-#  Run the job over the spawn plugin. Refer to the README for instructions on 
-#  installing Spawn. 
-#
-#  Spawn forks the entire process once for each job. This means that the job starts 
-#  with a very low latency, but takes up more memory for each job. 
-# 
-#  It's also possible to configure Spawn to start a Thread for each job. Do this
-#  by setting
-#
-#      Workling::Remote::Runners::SpawnRunner.options = { :method => :thread }
-#
-#  Have a look at the Spawn README to find out more about the characteristics of this. 
-#
-module Workling
-  module Remote
-    module Runners
-      class SpawnRunner < Workling::Remote::Runners::Base
-        cattr_accessor :options
-        
-        # use thread for development and test modes. easier to hunt down exceptions that way. 
-        @@options = { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :thread : :fork) }
-        include Spawn if Workling.spawn_installed?
-        
-        # dispatches to Spawn, using the :fork option. 
-        def run(clazz, method, options = {})
-          spawn(SpawnRunner.options) do # exceptions are trapped in here. 
-            dispatch!(clazz, method, options)
-          end
-          
-          return nil # that means nothing!
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/runners/starling_runner.rb

@@ -1,13 +0,0 @@
-require 'workling/remote/runners/client_runner'
-
-#
-#  DEPRECATED. Should use ClientRunner instead. 
-#
-module Workling
-  module Remote
-    module Runners
-      class StarlingRunner < Workling::Remote::Runners::ClientRunner
-      end
-    end
-  end
-end