elecnix-workling 0.3.1

data/lib/workling/remote.rb

@@ -0,0 +1,42 @@
+require "workling/remote/runners/not_remote_runner"
+require "workling/remote/runners/spawn_runner"
+require "workling/remote/runners/starling_runner"
+require "workling/remote/runners/backgroundjob_runner"
+
+require 'digest/md5'
+
+#
+#  Scoping Module for Runners. 
+#
+module Workling
+  module Remote
+    
+    # set the desired runner here. this is initialized with Workling.default_runner. 
+    mattr_accessor :dispatcher
+    
+    # set the desired invoker. this class grabs work from the job broker and executes it. 
+    mattr_accessor :invoker
+    @@invoker ||= Workling::Remote::Invokers::ThreadedPoller
+    
+    # retrieve the dispatcher or instantiate it using the defaults
+    def self.dispatcher
+      @@dispatcher ||= Workling.default_runner
+    end
+    
+    # generates a unique identifier for this particular job. 
+    def self.generate_uid(clazz, method)
+      uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
+      "#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
+    end
+    
+    # dispatches to a workling. writes the :uid for this work into the options hash, so make 
+    # sure you pass in a hash if you want write to a return store in your workling.
+    def self.run(clazz, method, options = {})
+      uid = Workling::Remote.generate_uid(clazz, method)
+      options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
+      Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist. 
+      dispatcher.run(clazz, method, options)
+      uid
+    end
+  end
+end

data/lib/workling/return/store/base.rb

@@ -0,0 +1,42 @@
+#
+#  Basic interface for getting and setting Data which needs to be passed between Workers and
+#  client code. 
+#
+module Workling
+  module Return
+    module Store
+      mattr_accessor :instance
+      
+      # set a value in the store with the given key. delegates to the returnstore. 
+      def self.set(key, value)
+        self.instance.set(key, value)
+      end
+      
+      # get a value from the store. this should be destructive. delegates to the returnstore. 
+      def self.get(key)
+        self.instance.get(key)
+      end
+      
+      #
+      #  Base Class for Return Stores. Subclasses need to implement set and get. 
+      #
+      class Base
+        
+        # set a value in the store with the given key. 
+        def set(key, value)
+          raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
+        end
+      
+        # get a value from the store. this should be destructive.
+        def get(key)
+          raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
+        end
+        
+        def iterator(key)
+          Workling::Return::Store::Iterator.new(key)
+        end
+        
+      end
+    end
+  end
+end

data/lib/workling/return/store/iterator.rb

@@ -0,0 +1,24 @@
+#
+#  Iterator class for iterating over return values. 
+#
+module Workling
+  module Return
+    module Store
+      class Iterator
+        
+        include Enumerable
+        
+        def initialize(uid)
+          @uid = uid
+        end
+        
+        def each
+          while item = Workling.return.get(@uid)
+            yield item
+          end
+        end
+        
+      end
+    end
+  end
+end

data/lib/workling/return/store/memory_return_store.rb

@@ -0,0 +1,26 @@
+require 'workling/return/store/base'
+
+#
+#  Stores directly into memory. This is for tests only - not for production use. aight?
+#
+module Workling
+  module Return
+    module Store
+      class MemoryReturnStore < Base
+        attr_accessor :sky
+        
+        def initialize
+          self.sky = Hash.new([])
+        end
+        
+        def set(key, value)
+          self.sky[key] << value
+        end
+        
+        def get(key)
+          self.sky[key].shift
+        end
+      end
+    end
+  end
+end

data/lib/workling/return/store/rudeq_return_store.rb

@@ -0,0 +1,24 @@
+require 'workling/return/store/base'
+require 'workling/rudeq/client'
+
+module Workling
+  module Return
+    module Store
+      class RudeqReturnStore < Base
+        cattr_accessor :client
+        
+        def initialize
+          self.class.client = Workling::Rudeq::Client.new
+        end
+        
+        def set(key, value)
+          self.class.client.set(key, value)
+        end
+        
+        def get(key)
+          self.class.client.get(key)
+        end
+      end
+    end
+  end
+end

data/lib/workling/return/store/starling_return_store.rb

@@ -0,0 +1,31 @@
+require 'workling/return/store/base'
+require 'workling/clients/memcache_queue_client'
+
+#
+#  Recommended Return Store if you are using the Starling Runner. This
+#  Simply sets and gets values against queues. 'key' is the name of the respective Queue. 
+#
+module Workling
+  module Return
+    module Store
+      class StarlingReturnStore < Base
+        cattr_accessor :client
+        
+        def initialize
+          self.client = Workling::Clients::MemcacheQueueClient.new
+          self.client.connect
+        end
+        
+        # set a value in the queue 'key'. 
+        def set(key, value)
+          self.class.client.set(key, value)
+        end
+        
+        # get a value from starling queue 'key'.
+        def get(key)
+          self.class.client.get(key)
+        end
+      end
+    end
+  end
+end

data/lib/workling/routing/base.rb

@@ -0,0 +1,13 @@
+#
+#  Base Class for Routing. Routing takes the worker method TestWorker#something, 
+#  and serializes the signature in some way. 
+#
+module Workling
+  module Routing
+    class Base < Hash
+      def method_name
+        raise Exception.new("method_name not implemented.")
+      end
+    end
+  end
+end

data/lib/workling/routing/class_and_method_routing.rb

@@ -0,0 +1,55 @@
+require 'workling/routing/base'
+
+#
+#  Holds a hash of routes. Each Worker method has a corresponding hash entry after building. 
+#
+module Workling
+  module Routing
+    class ClassAndMethodRouting < Base
+          
+      # initializes and builds routing hash. 
+      def initialize
+        super
+
+        build
+      end
+      
+      # returns the worker method name, given the routing string. 
+      def method_name(queue)
+        queue.split("__").last
+      end
+      
+      # returns the routing string, given a class and method. delegating. 
+      def queue_for(clazz, method)
+        ClassAndMethodRouting.queue_for(clazz, method)
+      end
+              
+      # returns the routing string, given a class and method.
+      def self.queue_for(clazz, method)
+        "#{ clazz.to_s.tableize }/#{ method }".split("/").join("__") # Don't split with : because it messes up memcache stats
+      end
+      
+      # returns all routed
+      def queue_names
+        self.keys
+      end
+      
+      # dare you to remove this! go on! 
+      def queue_names_routing_class(clazz)
+        self.select { |x, y|  y.is_a?(clazz) }.map { |x, y| x }
+      end
+      
+      private
+        def build
+          Workling::Discovery.discovered.each do |clazz|
+            methods = clazz.instance_methods(false)
+            methods.each do |method|
+              next if method == 'create'  # Skip the create method
+              queue =  queue_for(clazz, method)
+              self[queue] = clazz.new
+            end   
+          end     
+        end       
+    end
+  end
+end

data/lib/workling/rudeq/client.rb

@@ -0,0 +1,17 @@
+require 'workling/rudeq'
+
+module Workling
+  module Rudeq
+    class Client
+      attr_reader :queue
+      
+      def initialize 
+        @queue = Workling::Rudeq.config[:queue_class].constantize
+      end
+      
+      def method_missing(method, *args)
+        @queue.send(method, *args)
+      end
+    end
+  end
+end

data/lib/workling/rudeq/poller.rb

@@ -0,0 +1,116 @@
+require 'workling/rudeq'
+
+module Workling
+  module Rudeq
+    
+    class Poller
+      
+      cattr_accessor :sleep_time # Seconds to sleep before looping
+      cattr_accessor :reset_time # Seconds to wait while resetting connection
+
+      def initialize(routing)
+        Poller.sleep_time = Workling::Rudeq.config[:sleep_time] || 2
+        Poller.reset_time = Workling::Rudeq.config[:reset_time] || 30
+          
+        @routing = routing
+        @workers = ThreadGroup.new
+      end      
+      
+      def logger
+        Workling::Base.logger
+      end
+    
+      def listen
+                
+        # Allow concurrency for our tasks
+        ActiveRecord::Base.allow_concurrency = true
+
+        # Create a thread for each worker.
+        Workling::Discovery.discovered.each do |clazz|
+          logger.debug("Discovered listener #{clazz}")
+          @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
+        end
+        
+        # Wait for all workers to complete
+        @workers.list.each { |t| t.join }
+
+        # Clean up all the connections.
+        ActiveRecord::Base.verify_active_connections!
+      end
+      
+      # gracefully stop processing
+      def stop
+        @workers.list.each { |w| w[:shutdown] = true }
+      end
+      
+      ##
+      ## Thread procs
+      ##
+      
+      # Listen for one worker class
+      def clazz_listen(clazz)
+        
+        logger.debug("Listener thread #{clazz.name} started")
+           
+        # Read thread configuration if available
+        if Rudeq.config.has_key?(:listeners)
+          if Rudeq.config[:listeners].has_key?(clazz.to_s)
+            config = Rudeq.config[:listeners][clazz.to_s].symbolize_keys
+            thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
+          end
+        end
+
+        hread_sleep_time ||= self.class.sleep_time
+        
+        connection = Workling::Rudeq::Client.new
+        puts "** Starting Workling::Rudeq::Client for #{clazz.name} queue"
+        
+        # Start dispatching those messages
+        while (!Thread.current[:shutdown]) do
+          begin
+            
+            # Keep MySQL connection alive
+            unless ActiveRecord::Base.connection.active?
+              unless ActiveRecord::Base.connection.reconnect!
+                logger.fatal("FAILED - Database not available")
+                break
+              end
+            end
+
+            # Dispatch and process the messages
+            n = dispatch!(connection, clazz)
+            logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
+            sleep(self.class.sleep_time) unless n > 0
+          end
+        end
+        
+        logger.debug("Listener thread #{clazz.name} ended")
+      end
+      
+      # Dispatcher for one worker class.
+      # Returns the number of worker methods called
+      def dispatch!(connection, clazz)
+        n = 0
+        for queue in @routing.queue_names_routing_class(clazz)
+          begin
+            result = connection.get(queue)
+            if result
+              n += 1
+              handler = @routing[queue]
+              method_name = @routing.method_name(queue)
+              logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
+              handler.send(method_name, result)
+            end
+          rescue
+            logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
+            raise e
+          rescue Object => e
+            logger.error("FAILED to process queue #{ queue }. #{ @routing[queue] } could not handle invocation of #{ @routing.method_name(queue) } with #{ result.inspect }: #{ e }.\n#{ e.backtrace.join("\n") }")
+          end
+        end
+        
+        return n
+      end
+    end
+  end
+end

data/lib/workling/rudeq.rb

@@ -0,0 +1,7 @@
+module Workling
+  module Rudeq
+    def self.config
+      @@config ||=  {:queue_class => "RudeQueue"}
+    end    
+  end
+end

data/lib/workling.rb

@@ -0,0 +1,150 @@
+#
+#  I can haz am in your Workling are belong to us! 
+#
+module Workling
+  class WorklingError < StandardError; end
+  class WorklingNotFoundError < WorklingError; end
+  class WorklingConnectionError < WorklingError; end
+  class QueueserverNotFoundError < WorklingError
+    def initialize
+      super "config/workling.yml configured to connect to queue server on #{ Workling.config[:listens_on] } for this environment. could not connect to queue server on this host:port. for starling users: pass starling the port with -p flag when starting it. If you don't want to use Starling, then explicitly set Workling::Remote.dispatcher (see README for an example)"
+    end
+  end
+
+  class ConfigurationError < WorklingError
+    def initialize
+      super File.exist?(File.join(RAILS_ROOT, 'config', 'starling.yml')) ? 
+        "config/starling.yml has been depracated. rename your config file to config/workling.yml then try again!" :
+        "config/workling.yml could not be loaded. check out README.markdown to see what this file should contain. "
+    end
+  end
+  
+  mattr_accessor :load_path
+  @@load_path = [ File.expand_path(File.join(File.dirname(__FILE__), '../../../../app/workers')) ]
+  VERSION = "0.4.2.2"
+  
+  #
+  # determine the runner to use if nothing is specifically set. workling will try to detect
+  # starling, spawn, or bj, in that order. if none of these are found, notremoterunner will
+  # be used. 
+  #
+  # this can be overridden by setting Workling::Remote.dispatcher, eg:
+  #   Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new
+  #
+  def self.default_runner
+    if RAILS_ENV == "test"
+      Workling::Remote::Runners::NotRemoteRunner.new
+    elsif starling_installed?
+      Workling::Remote::Runners::StarlingRunner.new
+    elsif spawn_installed?
+      Workling::Remote::Runners::SpawnRunner.new
+    elsif bj_installed?
+      Workling::Remote::Runners::BackgroundjobRunner.new
+    else
+      Workling::Remote::Runners::NotRemoteRunner.new
+    end
+  end
+  
+  #
+  # gets the worker instance, given a class. the optional method argument will cause an 
+  # exception to be raised if the worker instance does not respoind to said method. 
+  #
+  def self.find(clazz, method = nil)
+    begin
+      inst = clazz.to_s.camelize.constantize.new 
+    rescue NameError
+      raise_not_found(clazz, method)
+    end
+    raise_not_found(clazz, method) if method && !inst.respond_to?(method)
+    inst
+  end
+  
+  # returns Workling::Return::Store.instance. 
+  def self.return
+    Workling::Return::Store.instance
+  end
+
+  # is spawn installed?
+  def self.spawn_installed?
+    begin
+      require 'spawn'
+    rescue LoadError
+    end
+
+    Object.const_defined? "Spawn"
+  end
+
+  # is starling installed?  
+  def self.starling_installed?
+    begin
+      require 'starling' 
+    rescue LoadError
+    end
+      
+    Object.const_defined? "Starling"
+  end
+
+  # is bj installed?
+  def self.bj_installed?
+    Object.const_defined? "Bj"
+  end
+  
+  # tries to load fiveruns-memcache-client. if this isn't found, 
+  # memcache-client is searched for. if that isn't found, don't do anything. 
+  def self.try_load_a_memcache_client
+    begin
+      gem 'fiveruns-memcache-client'
+      require 'memcache'
+    rescue Gem::LoadError
+      begin
+        gem 'memcache-client'
+        require 'memcache'
+      rescue Gem::LoadError
+        Workling::Base.logger.info "WORKLING: couldn't find a memcache client - you need one for the starling runner. "
+      end
+    end
+  end
+  
+  # attempts to load amqp and writes out descriptive error message if not present
+  def self.try_load_an_amqp_client
+    begin
+      require 'mq'
+    rescue Exception => e
+      raise WorklingError.new(
+        "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+        "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+      )
+    end
+  end
+  
+  #
+  #  returns a config hash. reads RAILS_ROOT/config/workling.yml
+  #
+  def self.config
+    begin
+      config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
+      @@config ||=  YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys
+      @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options]
+      @@config 
+    rescue
+       # config files could not be read correctly
+      raise ConfigurationError.new
+    end
+  end
+  
+  #
+  #  Raises exceptions thrown inside of the worker. normally, these are logged to 
+  #  logger.error. it's easy to miss these log calls while developing, though. 
+  #
+  mattr_accessor :raise_exceptions
+  @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
+  
+  def self.raise_exceptions?
+    @@raise_exceptions
+  end
+  
+  private
+    def self.raise_not_found(clazz, method)
+      raise Workling::WorklingNotFoundError.new("could not find #{ clazz }:#{ method } workling. ") 
+    end
+end