did_workling 0.0.1

data/lib/workling/base.rb

@@ -0,0 +1,59 @@
+#
+#  All worker classes must inherit from this class, and be saved in app/workers. 
+# 
+#  The Worker lifecycle: 
+#    The Worker is loaded once, at which point the instance method 'create' is called. 
+#
+#  Invoking Workers: 
+#    Calling async_my_method on the worker class will trigger background work.
+#    This means that the loaded Worker instance will receive a call to the method
+#    my_method(:uid => "thisjobsuid2348732947923"). 
+#
+#    The Worker method must have a single hash argument. Note that the job :uid will
+#    be merged into the hash. 
+#
+module Workling
+  class Base
+    cattr_accessor :logger
+    @@logger ||= ::RAILS_DEFAULT_LOGGER
+    
+    def self.inherited(subclass)
+      Workling::Discovery.discovered << subclass
+    end
+    
+    def initialize
+      super
+      
+      create
+    end
+
+    # Put worker initialization code in here. This is good for restarting jobs that
+    # were interrupted.
+    def create
+    end
+    
+    # takes care of suppressing remote errors but raising Workling::WorklingNotFoundError
+    # where appropriate. swallow workling exceptions so that everything behaves like remote code.
+    # otherwise StarlingRunner and SpawnRunner would behave too differently to NotRemoteRunner.
+    def dispatch_to_worker_method(method, options)
+      begin
+        self.send(method, options)
+      rescue Exception => e
+        raise e if e.kind_of?(Workling::WorklingError)
+        logger.error "WORKLING ERROR: runner could not invoke #{ self.class }:#{ method } with #{ options.inspect }. error was: #{ e.inspect }\n #{ e.backtrace.join("\n") }"
+
+        # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
+        raise e if Workling.raise_exceptions?
+      end
+    end    
+  
+    # thanks to blaine cook for this suggestion.
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^asynch?_(.*)/
+        Workling::Remote.run(self.to_s.dasherize, $1, *args)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/workling/clients/amqp_client.rb

@@ -0,0 +1,40 @@
+require 'workling/clients/base'
+Workling.try_load_an_amqp_client
+
+#
+#  An Ampq client
+#
+module Workling
+  module Clients
+    class AmqpClient < Workling::Clients::Base
+      
+      # starts the client. 
+      def connect
+        begin
+          @amq = MQ.new
+        rescue
+          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+        end
+      end
+      
+      # no need for explicit closing. when the event loop
+      # terminates, the connection is closed anyway. 
+      def close; true; end
+      
+      # subscribe to a queue
+      def subscribe(key)
+        @amq.queue(key).subscribe do |data|
+          value = Marshal.load(data)
+          yield value
+        end
+      end
+      
+      # request and retrieve work
+      def retrieve(key); @amq.queue(key); end
+      def request(key, value)
+        data = Marshal.dump(value)
+        @amq.queue(key).publish(data)
+      end
+    end
+  end
+end

data/lib/workling/clients/base.rb

@@ -0,0 +1,54 @@
+#
+#  Clients are responsible for communicating with a job broker (ie connecting to starling or rabbitmq.)
+#
+#  Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
+#  from a specific type of job. 
+#
+module Workling
+  module Clients
+    class Base
+
+      #
+      #  Requests a job on the broker.
+      #
+      #      work_type: 
+      #      arguments: the argument to the worker method
+      #
+      def request(work_type, arguments)
+        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
+      end    
+
+      #
+      #  Gets job results off a job broker. Returns nil if there are no results. 
+      #
+      #      worker_uid: the uid returned by workling when the work was dispatched
+      #
+      def retrieve(work_uid)
+        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+      end
+      
+      #
+      #  Subscribe to job results in a job broker.
+      #
+      #      worker_type: 
+      #
+      def subscribe(work_type)
+        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+      end
+      
+      #
+      #  Opens a connection to the job broker.
+      #
+      def connect
+        raise NotImplementedError.new("Implement connect() in your client. ")
+      end
+      
+      #
+      #  Closes the connection to the job broker. 
+      #
+      def close
+        raise NotImplementedError.new("Implement close() in your client. ")
+      end
+    end
+  end
+end

data/lib/workling/clients/memcache_queue_client.rb

@@ -0,0 +1,82 @@
+require 'workling/clients/base'
+
+#
+#  This client can be used for all Queue Servers that speak Memcached, such as Starling. 
+#
+#  Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client, 
+#  or memcache-client, if this is not available. See the README for a discussion of the memcache 
+#  clients. 
+#
+#  method_missing delegates all messages through to the underlying memcache connection. 
+#
+module Workling
+  module Clients
+    class MemcacheQueueClient < Workling::Clients::Base
+      
+      # the class with which the connection is instantiated
+      cattr_accessor :memcache_client_class
+      @@memcache_client_class ||= ::MemCache
+      
+      # the url with which the memcache client expects to reach starling
+      attr_accessor :queueserver_urls
+      
+      # the memcache connection object
+      attr_accessor :connection
+      
+      #
+      #  the client attempts to connect to queueserver using the configuration options found in 
+      #
+      #      Workling.config. this can be configured in config/workling.yml. 
+      #
+      #  the initialization code will raise an exception if memcache-client cannot connect 
+      #  to queueserver.
+      #
+      def connect
+        @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url }
+        options = [@queueserver_urls, Workling.config[:memcache_options]].compact
+        self.connection = MemcacheQueueClient.memcache_client_class.new(*options)
+        
+        raise_unless_connected!
+      end
+      
+      # closes the memcache connection
+      def close
+        self.connection.flush_all
+        self.connection.reset
+      end
+
+      # implements the client job request and retrieval 
+      def request(key, value)
+        set(key, value)
+      end
+      
+      def retrieve(key)
+        begin
+          get(key)
+        rescue MemCache::MemCacheError => e
+          # failed to enqueue, raise a workling error so that it propagates upwards
+          raise Workling::WorklingError.new("#{e.class.to_s} - #{e.message}")        
+        end
+      end
+            
+      private
+        # make sure we can actually connect to queueserver on the given port
+        def raise_unless_connected!
+          begin 
+            self.connection.stats
+          rescue
+            raise Workling::QueueserverNotFoundError.new
+          end
+        end
+        
+        # delegates directly through to the memcache connection. 
+        def method_missing(method, *args)
+          begin
+            self.connection.send(method, *args)
+          rescue MemCache::MemCacheError => e
+            raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
+          end
+        end
+    end
+  end
+end

data/lib/workling/discovery.rb

@@ -0,0 +1,14 @@
+#
+#  Discovery is responsible for loading workers in app/workers. 
+#
+module Workling
+  class Discovery
+    cattr_accessor :discovered
+    @@discovered = []
+    
+    # requires worklings so that they are added to routing. 
+    def self.discover!
+      Dir.glob(Workling.load_path.map { |p| "#{ p }/**/*.rb" }).each { |wling| require wling }
+    end
+  end
+end

data/lib/workling/remote.rb

@@ -0,0 +1,42 @@
+require "workling/remote/runners/not_remote_runner"
+require "workling/remote/runners/spawn_runner"
+require "workling/remote/runners/starling_runner"
+require "workling/remote/runners/backgroundjob_runner"
+
+require 'digest/md5'
+
+#
+#  Scoping Module for Runners. 
+#
+module Workling
+  module Remote
+    
+    # set the desired runner here. this is initialized with Workling.default_runner. 
+    mattr_accessor :dispatcher
+    
+    # set the desired invoker. this class grabs work from the job broker and executes it. 
+    mattr_accessor :invoker
+    @@invoker ||= Workling::Remote::Invokers::ThreadedPoller
+    
+    # retrieve the dispatcher or instantiate it using the defaults
+    def self.dispatcher
+      @@dispatcher ||= Workling.default_runner
+    end
+    
+    # generates a unique identifier for this particular job. 
+    def self.generate_uid(clazz, method)
+      uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
+      "#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
+    end
+    
+    # dispatches to a workling. writes the :uid for this work into the options hash, so make 
+    # sure you pass in a hash if you want write to a return store in your workling.
+    def self.run(clazz, method, options = {})
+      uid = Workling::Remote.generate_uid(clazz, method)
+      options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
+      Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist. 
+      dispatcher.run(clazz, method, options)
+      uid
+    end
+  end
+end

data/lib/workling/remote/invokers/base.rb

@@ -0,0 +1,124 @@
+#
+#  Invokers are responsible for 
+#
+#      1. grabbing work off a job broker (such as a starling or rabbitmq server).
+#      2. routing (mapping) that work onto the correct worker method. 
+#      3.invoking the worker method, passing any arguments that came off the broker.
+#
+#   Invokers should implement their own concurrency strategies. For example, 
+#   The there is a ThreadedPoller which starts a thread for each Worker class.
+#
+#   This base Invoker class defines the methods an Invoker needs to implement. 
+#  
+module Workling
+  module Remote
+    module Invokers
+      class Base
+        
+        attr_accessor :sleep_time, :reset_time
+        
+        #
+        #  call up with super in the subclass constructor.
+        #
+        def initialize(routing, client_class)
+          @routing = routing
+          @client_class = client_class
+          @sleep_time = Workling.config[:sleep_time] || 2
+          @reset_time = Workling.config[:reset_time] || 30
+          @@mutex ||= Mutex.new
+        end
+        
+        #
+        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
+        #
+        def listen
+          raise NotImplementedError.new("Implement listen() in your Invoker. ")
+        end        
+        
+        #
+        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
+        #  to finish. 
+        #
+        def stop
+          raise NotImplementedError.new("Implement stop() in your Invoker. ")
+        end
+        
+        # 
+        #  Runs the worker method, given
+        #
+        #      type: the worker route
+        #      args: the arguments to be passed into the worker method.
+        #
+        def run(type, args)
+          worker = @routing[type]
+          method = @routing.method_name(type)
+          worker.dispatch_to_worker_method(method, args)
+        end
+        
+        # returns the Workling::Base.logger
+        def logger; Workling::Base.logger; end
+        
+        protected
+        
+          # handle opening and closing of client. pass code block to this method. 
+          def connect
+            @client = @client_class.new
+            @client.connect
+            
+            begin
+              yield
+            ensure
+              @client.close
+              ActiveRecord::Base.verify_active_connections!
+            end
+          end
+
+          #
+          #  Loops through the available routes, yielding for each route. 
+          #  This continues until @shutdown is set on this instance. 
+          #
+          def loop_routes
+            while(!@shutdown) do
+              ensure_activerecord_connection
+              
+              routes.each do |route|
+                break if @shutdown
+                yield route
+              end
+              
+              sleep self.sleep_time
+            end
+          end
+
+          #
+          #  Returns the complete set of active routes
+          #
+          def routes
+            @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
+          end
+          
+          # Thanks for this Brent! 
+          #
+          #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
+          #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
+          #     to wrap the code that checks for a connection in in a mutex.
+          #
+          #     ....I noticed this while working with a multi-core machine that 
+          #     was spawning multiple workling threads. Some of my workling 
+          #     threads would hit serious issues at this block of code without 
+          #     the mutex.            
+          #
+          def ensure_activerecord_connection
+            @@mutex.synchronize do 
+              unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
+                unless ActiveRecord::Base.connection.reconnect!
+                  logger.fatal("Failed - Database not available!")
+                  break
+                end
+              end
+            end            
+          end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/basic_poller.rb

@@ -0,0 +1,41 @@
+require 'workling/remote/invokers/base'
+
+#
+#  A basic polling invoker. 
+#  
+module Workling
+  module Remote
+    module Invokers
+      class BasicPoller < Workling::Remote::Invokers::Base
+        
+        #
+        #  set up client, sleep time
+        #
+        def initialize(routing, client_class)
+          super
+        end
+        
+        #
+        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
+        #
+        def listen
+          connect do
+            loop_routes do |route|
+              if args = @client.retrieve(route)
+                run(route, args)
+              end
+            end
+          end
+        end
+        
+        #
+        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
+        #  to finish. 
+        #
+        def stop
+          @shutdown = true
+        end
+      end
+    end
+  end
+end