did_workling 0.0.1

data/lib/workling/remote/invokers/eventmachine_subscriber.rb

@@ -0,0 +1,37 @@
+require 'eventmachine'
+require 'workling/remote/invokers/base'
+
+#
+#  Subscribes the workers to the correct queues. 
+# 
+module Workling
+  module Remote
+    module Invokers
+      class EventmachineSubscriber < Workling::Remote::Invokers::Base
+        
+        def initialize(routing, client_class)
+          super
+        end
+        
+        #
+        #  Starts EM loop and sets up subscription callbacks for workers. 
+        #
+        def listen
+          EM.run do
+            connect do
+              routes.each do |route|
+                @client.subscribe(route) do |args|
+                  run(route, args)
+                end
+              end
+            end
+          end
+        end
+                
+        def stop
+          EM.stop if EM.reactor_running?
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/threaded_poller.rb

@@ -0,0 +1,149 @@
+# -*- coding: utf-8 -*-
+require 'workling/remote/invokers/base'
+
+#
+#  A threaded polling Invoker. 
+# 
+#  TODO: refactor this to make use of the base class. 
+# 
+module Workling
+  module Remote
+    module Invokers
+      class ThreadedPoller < Workling::Remote::Invokers::Base
+        
+        cattr_accessor :sleep_time, :reset_time
+      
+        def initialize(routing, client_class)
+          super
+          
+          ThreadedPoller.sleep_time = Workling.config[:sleep_time] || 2
+          ThreadedPoller.reset_time = Workling.config[:reset_time] || 30
+          
+          @workers = ThreadGroup.new
+          @mutex = Mutex.new
+        end      
+          
+        def listen                
+          # Allow concurrency for our tasks
+          ActiveRecord::Base.allow_concurrency = true
+
+          # Create a thread for each worker.
+          Workling::Discovery.discovered.each do |clazz|
+            logger.debug("Discovered listener #{clazz}")
+            @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
+          end
+          
+          # Wait for all workers to complete
+          @workers.list.each { |t| t.join }
+
+          logger.debug("Reaped listener threads. ")
+        
+          # Clean up all the connections.
+          ActiveRecord::Base.verify_active_connections!
+          logger.debug("Cleaned up connection: out!")
+        end
+      
+        # Check if all Worker threads have been started. 
+        def started?
+          logger.debug("checking if started... list size is #{ worker_threads }")
+          Workling::Discovery.discovered.size == worker_threads
+        end
+        
+        # number of worker threads running
+        def worker_threads
+          @workers.list.size
+        end
+      
+        # Gracefully stop processing
+        def stop
+          logger.info("stopping threaded poller...")
+          sleep 1 until started? # give it a chance to start up before shutting down. 
+          logger.info("Giving Listener Threads a chance to shut down. This may take a while... ")
+          @workers.list.each { |w| w[:shutdown] = true }
+          logger.info("Listener threads were shut down.  ")
+        end
+
+        # Listen for one worker class
+        def clazz_listen(clazz)
+          logger.debug("Listener thread #{clazz.name} started")
+           
+          # Read thread configuration if available
+          if Workling.config.has_key?(:listeners)
+            if Workling.config[:listeners].has_key?(clazz.to_s)
+              config = Workling.config[:listeners][clazz.to_s].symbolize_keys
+              thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
+            end
+          end
+
+          hread_sleep_time ||= self.class.sleep_time
+                
+          # Setup connection to client (one per thread)
+          connection = @client_class.new
+          connection.connect
+          logger.info("** Starting client #{ connection.class } for #{clazz.name} queue")
+     
+          # Start dispatching those messages
+          while (!Thread.current[:shutdown]) do
+            begin
+            
+              # Thanks for this Brent! 
+              #
+              #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
+              #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
+              #     to wrap the code that checks for a connection in in a mutex.
+              #
+              #     ....I noticed this while working with a multi-core machine that 
+              #     was spawning multiple workling threads. Some of my workling 
+              #     threads would hit serious issues at this block of code without 
+              #     the mutex.            
+              #
+              @mutex.synchronize do 
+                ActiveRecord::Base.connection.verify!  # Keep MySQL connection alive
+                unless ActiveRecord::Base.connection.active?
+                  logger.fatal("Failed - Database not available!")
+                end
+              end
+
+              # Dispatch and process the messages
+              n = dispatch!(connection, clazz)
+              logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
+              sleep(self.class.sleep_time) unless n > 0
+            
+              # If there is a memcache error, hang for a bit to give it a chance to fire up again
+              # and reset the connection.
+              rescue Workling::WorklingConnectionError
+                logger.warn("Listener thread #{clazz.name} failed to connect. Resetting connection.")
+                sleep(self.class.reset_time)
+                connection.reset
+            end
+          end
+        
+          logger.debug("Listener thread #{clazz.name} ended")
+        end
+      
+        # Dispatcher for one worker class. Will throw MemCacheError if unable to connect.
+        # Returns the number of worker methods called
+        def dispatch!(connection, clazz)
+          n = 0
+          for queue in @routing.queue_names_routing_class(clazz)
+            begin
+              result = connection.retrieve(queue)
+              if result
+                n += 1
+                handler = @routing[queue]
+                method_name = @routing.method_name(queue)
+                logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
+                handler.dispatch_to_worker_method(method_name, result)
+              end
+            rescue MemCache::MemCacheError => e
+              logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
+              raise e
+            end
+          end
+        
+          return n
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/backgroundjob_runner.rb

@@ -0,0 +1,35 @@
+require 'workling/remote/runners/base'
+
+#
+#  Use Ara Howards BackgroundJob to run the work. BackgroundJob loads Rails once per requested Job. 
+#  It persists over the database, and there is no requirement for separate processes to be started. 
+#  Since rails has to load before each request, it takes a moment for the job to run. 
+#
+module Workling
+  module Remote
+    module Runners
+      class BackgroundjobRunner < Workling::Remote::Runners::Base
+        cattr_accessor :routing
+        
+        def initialize
+          BackgroundjobRunner.routing = 
+            Workling::Routing::ClassAndMethodRouting.new
+        end
+        
+        #  passes the job to bj by serializing the options to xml and passing them to
+        #  ./script/bj_invoker.rb, which in turn routes the deserialized args to the
+        #  appropriate worker. 
+        def run(clazz, method, options = {})
+          stdin = @@routing.queue_for(clazz, method) + 
+                  " " + 
+                  options.to_xml(:indent => 0, :skip_instruct => true)
+                  
+          Bj.submit "./script/runner ./script/bj_invoker.rb", 
+            :stdin => stdin
+          
+          return nil # that means nothing!
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/base.rb

@@ -0,0 +1,42 @@
+#
+#  Base class for Workling Runners.
+#
+#  Runners must subclass this and implement the method 
+#
+#     Workling::Remote::Runners::Base#run(clazz, method, options = {})
+#
+#  which is responsible for pushing the requested job into the background. Depending
+#  on the Runner, this may require other code to dequeue the job. The actual
+#  invocation of the runner should be done like this: 
+#
+#      Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+#
+#  This ensures for consistent logging and handling of propagated exceptions. You can
+#  also call the convenience method 
+#       
+#      Workling::Remote::Runners::Base#dispatch!(clazz, method, options)
+#
+#  which invokes this for you. 
+#
+module Workling
+  module Remote
+    module Runners
+      class Base
+        
+        # runner uses this to connect to a job broker
+        cattr_accessor :client
+        
+        # default logger defined in Workling::Base.logger
+        def logger
+          Workling::Base.logger
+        end
+
+        # find the worker instance and invoke it. Invoking the worker method like this ensures for 
+        # consistent logging and handling of propagated exceptions. 
+        def dispatch!(clazz, method, options)
+          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/client_runner.rb

@@ -0,0 +1,45 @@
+require 'workling/remote/runners/base'
+require 'workling/clients/memcache_queue_client'
+
+#
+#  Runs Jobs over a Client. The client should be a subclass of Workling::Client::Base. 
+#  Set the client like this: 
+#
+#      Workling::Remote::Runners::ClientRunner.client = Workling::Clients::AmqpClient.new
+#
+#  Jobs are dispatched by requesting them on the Client. The Runner takes care of mapping of queue names to worker code. 
+#  this is done with Workling::ClassAndMethodRouting, but you can use your own by sublassing Workling::Routing. 
+#  Don’t worry about any of this if you’re not dealing directly with the queues.
+#
+#  There’s a workling-client daemon that uses the configured invoker to retrieve work and dispatching these to the 
+#  responsible workers. If you intend to run this on a remote machine, then just check out your rails project 
+#  there and start up the workling client like this: ruby script/workling_client run.
+#
+module Workling
+  module Remote
+    module Runners
+      class ClientRunner < Workling::Remote::Runners::Base
+        
+        # Routing class. Workling::Routing::ClassAndMethodRouting.new by default. 
+        cattr_accessor :routing
+        @@routing ||= Workling::Routing::ClassAndMethodRouting.new
+        
+        # The workling Client class. Workling::Clients::MemcacheQueueClient.new by default. 
+        cattr_accessor :client
+        @@client ||= Workling::Clients::MemcacheQueueClient.new
+        
+        # enqueues the job onto the client
+        def run(clazz, method, options = {})
+          
+          # neet to connect in here as opposed to the constructor, since the EM loop is
+          # not available there. 
+          @connected ||= self.class.client.connect
+          
+          self.class.client.request(@@routing.queue_for(clazz, method), options)    
+          
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/not_remote_runner.rb

@@ -0,0 +1,23 @@
+require 'workling/remote/runners/base'
+
+#
+# directly dispatches to the worker method, in-process. options are first marshalled then dumped
+# in order to simulate the sideeffects of a remote call.
+#
+module Workling
+  module Remote
+    module Runners
+      class NotRemoteRunner < Workling::Remote::Runners::Base
+        
+        # directly dispatches to the worker method, in-process. options are first marshalled then dumped
+        # in order to simulate the sideeffects of a remote call. 
+        def run(clazz, method, options = {})
+          options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
+          dispatch!(clazz, method, options) 
+          
+          return nil # nada. niente.
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/spawn_runner.rb

@@ -0,0 +1,38 @@
+require 'workling/remote/runners/base'
+
+#
+#  Run the job over the spawn plugin. Refer to the README for instructions on 
+#  installing Spawn. 
+#
+#  Spawn forks the entire process once for each job. This means that the job starts 
+#  with a very low latency, but takes up more memory for each job. 
+# 
+#  It's also possible to configure Spawn to start a Thread for each job. Do this
+#  by setting
+#
+#      Workling::Remote::Runners::SpawnRunner.options = { :method => :thread }
+#
+#  Have a look at the Spawn README to find out more about the characteristics of this. 
+#
+module Workling
+  module Remote
+    module Runners
+      class SpawnRunner < Workling::Remote::Runners::Base
+        cattr_accessor :options
+        
+        # use thread for development and test modes. easier to hunt down exceptions that way. 
+        @@options = { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :thread : :fork) }
+        include Spawn if Workling.spawn_installed?
+        
+        # dispatches to Spawn, using the :fork option. 
+        def run(clazz, method, options = {})
+          spawn(SpawnRunner.options) do # exceptions are trapped in here. 
+            dispatch!(clazz, method, options)
+          end
+          
+          return nil # that means nothing!
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/runners/starling_runner.rb

@@ -0,0 +1,13 @@
+require 'workling/remote/runners/client_runner'
+
+#
+#  DEPRECATED. Should use ClientRunner instead. 
+#
+module Workling
+  module Remote
+    module Runners
+      class StarlingRunner < Workling::Remote::Runners::ClientRunner
+      end
+    end
+  end
+end

data/lib/workling/return/store/base.rb

@@ -0,0 +1,37 @@
+#
+#  Basic interface for getting and setting Data which needs to be passed between Workers and
+#  client code. 
+#
+module Workling
+  module Return
+    module Store
+      mattr_accessor :instance
+      
+      # set a value in the store with the given key. delegates to the returnstore. 
+      def self.set(key, value)
+        self.instance.set(key, value)
+      end
+      
+      # get a value from the store. this should be destructive. delegates to the returnstore. 
+      def self.get(key)
+        self.instance.get(key)
+      end
+      
+      #
+      #  Base Class for Return Stores. Subclasses need to implement set and get. 
+      #
+      class Base
+        
+        # set a value in the store with the given key. 
+        def set(key, value)
+          raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
+        end
+      
+        # get a value from the store. this should be destructive.
+        def get(key)
+          raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
+        end
+      end
+    end
+  end
+end

data/lib/workling/return/store/memory_return_store.rb

@@ -0,0 +1,26 @@
+require 'workling/return/store/base'
+
+#
+#  Stores directly into memory. This is for tests only - not for production use. aight?
+#
+module Workling
+  module Return
+    module Store
+      class MemoryReturnStore < Base
+        attr_accessor :sky
+        
+        def initialize
+          self.sky = {}
+        end
+        
+        def set(key, value)
+          self.sky[key] = value
+        end
+        
+        def get(key)
+          self.sky.delete(key)
+        end
+      end
+    end
+  end
+end