derfred-workling 0.4.9.2 → 0.4.9.3

data/lib/workling/clients/base.rb

@@ -1,5 +1,7 @@
 #
-#  Clients are responsible for communicating with a job broker (ie connecting to starling or rabbitmq.)
+#  Clients are responsible for dispatching jobs either to a broker (starling, rabbitmq etc) or invoke them (spawn)
+#
+#  Clients that involve a broker should subclass Workling::Clients::BrokerBase
 #
 #  Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
 #  from a specific type of job. 
@@ -8,9 +10,39 @@ module Workling
   module Clients
     class Base
 
+      #
+      # Load the required libraries, for this client
+      #
+      def self.load
+        
+      end
+
+
+      #
+      # See if the libraries required for this client are installed
+      #
+      def self.installed?
+        true
+      end
+
+
       # returns the Workling::Base.logger
       def logger; Workling::Base.logger; end
 
+
+      #
+      # Dispatch a job to the client. If this client uses a job broker, then
+      # this method should submit it, otherwise it should run the job
+      #
+      #    clazz: Name of the worker class
+      #    method: Name of the methods on the worker
+      #    options: optional arguments for the job
+      #
+      def dispatch(clazz, method, options = {})
+        raise NotImplementedError.new("Implement dispatch(clazz, method, options) in your client. ")
+      end
+
+
       #
       #  Requests a job on the broker.
       #
@@ -18,8 +50,8 @@ module Workling
       #      arguments: the argument to the worker method
       #
       def request(work_type, arguments)
-        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
-      end    
+        raise WorklingError.new("This client does not involve a broker.")
+      end
 
       #
       #  Gets job results off a job broker. Returns nil if there are no results. 
@@ -27,7 +59,7 @@ module Workling
       #      worker_uid: the uid returned by workling when the work was dispatched
       #
       def retrieve(work_uid)
-        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+        raise WorklingError.new("This client does not involve a broker.")
       end
       
       #
@@ -36,22 +68,22 @@ module Workling
       #      worker_type: 
       #
       def subscribe(work_type)
-        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+        raise WorklingError.new("This client does not involve a broker.")
       end
       
       #
       #  Opens a connection to the job broker.
       #
       def connect
-        raise NotImplementedError.new("Implement connect() in your client. ")
+        raise WorklingError.new("This client does not involve a broker.")
       end
       
       #
       #  Closes the connection to the job broker. 
       #
       def close
-        raise NotImplementedError.new("Implement close() in your client. ")
+        raise WorklingError.new("This client does not involve a broker.")
       end
     end
   end
-end
+end

data/lib/workling/clients/broker_base.rb

@@ -0,0 +1,63 @@
+module Workling
+  module Clients
+    class BrokerBase < Base
+
+      #
+      # Dispatch a job to the client. If this client uses a job broker, then
+      # this method should submit it, otherwise it should run the job
+      #
+      #    clazz: Name of the worker class
+      #    method: Name of the methods on the worker
+      #    options: optional arguments for the job
+      #
+      def dispatch(clazz, method, options = {})
+        @connected ||= connect
+        request(Workling::Remote.routing.queue_for(clazz, method), options)
+        return nil
+      end
+
+
+      #
+      #  Requests a job on the broker.
+      #
+      #      work_type: 
+      #      arguments: the argument to the worker method
+      #
+      def request(work_type, arguments)
+        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
+      end    
+
+      #
+      #  Gets job results off a job broker. Returns nil if there are no results. 
+      #
+      #      worker_uid: the uid returned by workling when the work was dispatched
+      #
+      def retrieve(work_uid)
+        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+      end
+      
+      #
+      #  Subscribe to job results in a job broker.
+      #
+      #      worker_type: 
+      #
+      def subscribe(work_type)
+        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+      end
+      
+      #
+      #  Opens a connection to the job broker.
+      #
+      def connect
+        raise NotImplementedError.new("Implement connect() in your client. ")
+      end
+      
+      #
+      #  Closes the connection to the job broker. 
+      #
+      def close
+        raise NotImplementedError.new("Implement close() in your client. ")
+      end
+    end
+  end
+end

data/lib/workling/clients/memcache_queue_client.rb

@@ -1,6 +1,3 @@
-require 'workling/clients/base'
-require 'memcache'
-
 #
 #  This client can be used for all Queue Servers that speak Memcached, such as Starling. 
 #
@@ -12,12 +9,26 @@ require 'memcache'
 #
 module Workling
   module Clients
-    class MemcacheQueueClient < Workling::Clients::Base
-      
-      # the class with which the connection is instantiated
-      cattr_accessor :memcache_client_class
-      @@memcache_client_class ||= ::MemCache
-      
+    class MemcacheQueueClient < Workling::Clients::BrokerBase
+
+      def self.installed?
+        begin
+          require 'starling' 
+        rescue LoadError
+        end
+
+        Object.const_defined? "Starling"
+      end
+
+      def self.load
+        begin
+          gem 'memcache-client'
+          require 'memcache'
+        rescue Gem::LoadError
+          Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
+        end
+      end
+
       # the url with which the memcache client expects to reach starling
       attr_accessor :queueserver_urls
       
@@ -35,11 +46,11 @@ module Workling
       def connect
         @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url }
         options = [@queueserver_urls, Workling.config[:memcache_options]].compact
-        self.connection = MemcacheQueueClient.memcache_client_class.new(*options)
-        
+        self.connection = ::MemCache.new(*options)
+
         raise_unless_connected!
       end
-      
+
       # closes the memcache connection
       def close
         self.connection.flush_all
@@ -86,6 +97,7 @@ module Workling
             raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
           end
         end
+
     end
   end
 end

data/lib/workling/clients/memory_queue_client.rb

@@ -0,0 +1,34 @@
+module Workling
+  module Clients
+    class MemoryQueueClient < Workling::Clients::BrokerBase
+
+      def initialize
+        @subscribers ||= {}
+        @queues ||= {}
+      end
+
+      # collects the worker blocks in a hash
+      def subscribe(work_type, &block)
+        @subscribers[work_type] = block
+      end
+
+      # immediately invokes the required worker block
+      def request(work_type, arguments)
+        if subscription = @subscribers[work_type]
+          subscription.call(arguments)
+        else
+          @queues[work_type] ||= []
+          @queues[work_type] << arguments
+        end
+      end
+
+      def retrieve(work_type)
+        queue = @queues[work_type]
+        queue.pop if queue
+      end
+
+      def connect; true; end
+      def close; true; end
+    end
+  end
+end

data/lib/workling/clients/not_client.rb

@@ -0,0 +1,14 @@
+#
+#  Dont actually run the job...
+#
+module Workling
+  module Clients
+    class NotClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        return nil # nada. niente.
+      end
+
+    end
+  end
+end

data/lib/workling/clients/not_remote_client.rb

@@ -0,0 +1,17 @@
+#
+#  Run the job inline
+#
+module Workling
+  module Clients
+    class NotRemoteClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
+        Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+
+        return nil # nada. niente.
+      end
+
+    end
+  end
+end

data/lib/workling/clients/spawn_client.rb

@@ -0,0 +1,47 @@
+#
+#  Run the job over the spawn plugin. Refer to the README for instructions on 
+#  installing Spawn. 
+#
+#  Spawn forks the entire process once for each job. This means that the job starts 
+#  with a very low latency, but takes up more memory for each job. 
+# 
+#  It's also possible to configure Spawn to start a Thread for each job. Do this
+#  by setting
+#
+#      Workling::Clients::SpawnClient.options = { :method => :thread }
+#
+#  Have a look at the Spawn README to find out more about the characteristics of this. 
+#
+module Workling
+  module Clients
+    class SpawnClient < Workling::Clients::Base
+
+      def self.installed?
+        begin
+          require 'spawn'
+        rescue LoadError
+        end
+
+        Object.const_defined? "Spawn"
+      end
+
+      cattr_writer :options
+      def options
+        return @@options if defined?(@@options)
+        # use thread for development and test modes. easier to hunt down exceptions that way.
+        @@options = { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :fork : :thread) }
+      end
+
+      include Spawn if installed?
+
+      def dispatch(clazz, method, options = {})
+        spawn(SpawnRunner.options) do # exceptions are trapped in here. 
+          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+        end
+
+        return nil # that means nothing!
+      end
+
+    end
+  end
+end

data/lib/workling/clients/sqs_client.rb

@@ -1,4 +1,3 @@
-require 'workling/clients/base'
 require 'json'
 require 'right_aws'
 
@@ -27,15 +26,17 @@ require 'right_aws'
 #   
 module Workling
   module Clients
-    class SqsClient < Workling::Clients::Base
+    class SqsClient < Workling::Clients::BrokerBase
 
-      AWS_MAX_QUEUE_NAME = 80
+      unless defined?(AWS_MAX_QUEUE_NAME)
+        AWS_MAX_QUEUE_NAME = 80
 
-      # Note that 10 is the maximum number of messages that can be retrieved
-      # in a single request.
-      DEFAULT_MESSAGES_PER_REQ = 10
-      DEFAULT_VISIBILITY_TIMEOUT = 30
-      DEFAULT_VISIBILITY_RESERVE = 10
+        # Note that 10 is the maximum number of messages that can be retrieved
+        # in a single request.
+        DEFAULT_MESSAGES_PER_REQ = 10
+        DEFAULT_VISIBILITY_TIMEOUT = 30
+        DEFAULT_VISIBILITY_RESERVE = 10
+      end
       
       # Mainly exposed for testing purposes
       attr_reader :sqs_options

data/lib/workling/clients/thread_client.rb

@@ -0,0 +1,18 @@
+#
+#  Spawns a Thread. Used for Tests only, to simulate a remote runner more realistically. 
+#
+module Workling
+  module Clients
+    class ThreadClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        Thread.new {
+          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+        }
+
+        return nil
+      end
+
+    end
+  end
+end

data/lib/workling/clients/xmpp_client.rb

@@ -1,6 +1,3 @@
-require 'workling/clients/base'
-Workling.try_load_xmpp4r
-
 #
 #  An XMPP client
 #
@@ -43,7 +40,20 @@ Workling.try_load_xmpp4r
 
 module Workling
   module Clients
-    class XmppClient < Workling::Clients::Base
+    class XmppClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          gem "xmpp4r"
+          require 'xmpp4r'
+          require "xmpp4r/pubsub"
+          require "xmpp4r/pubsub/helper/servicehelper.rb"
+          require "xmpp4r/pubsub/helper/nodebrowser.rb"
+          require "xmpp4r/pubsub/helper/nodehelper.rb"
+        rescue Exception => e
+          raise WorklingError.new("Couldnt load the XMPP library. check that you have the xmpp4r gem installed")
+        end
+      end
 
       # starts the client. 
       def connect

data/lib/workling/discovery.rb

@@ -4,7 +4,7 @@
 module Workling
   class Discovery
     cattr_accessor :discovered
-    @@discovered = []
+    @@discovered ||= []
 
     # requires worklings so that they are added to routing.
     def self.discover!

data/lib/workling/invokers/amqp_single_subscriber.rb

@@ -0,0 +1,42 @@
+require 'eventmachine'
+
+#
+# TODO - Subscribes a single worker to a single queue 
+# 
+module Workling
+  module Invokers
+    class AmqpSingleSubscriber < Workling::Invokers::Base
+
+      def initialize(routing, client_class)
+        super
+      end
+
+      #
+      # Starts EM loop and sets up subscription callback for the worker
+      # Create the queue and bind to exchange using the routing key
+      #
+      def listen
+        EM.run do
+          connect do
+            queue_name = @routing.queue_for
+            routing_key = @routing.routing_key_for
+      
+            # temp stuff to hook the queues and exchanges up
+            # wildcard routing - # (match all)
+            exch = MQ.topic
+            q = MQ.queue(queue_name)
+            q.bind(exch, :key => routing_key)
+      
+            @client.subscribe(queue_name) do |args|
+              run(queue_name, args)
+            end
+          end
+        end
+      end
+
+      def stop
+        EM.stop if EM.reactor_running?
+      end
+    end
+  end
+end