workling 0.4.9.7

data/lib/workling/clients/amqp_client.rb

@@ -0,0 +1,51 @@
+#
+#  An Ampq client
+#
+module Workling
+  module Clients
+    class AmqpClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          require 'mq'
+        rescue Exception => e
+          raise WorklingError.new(
+            "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+            "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+          )
+        end
+      end
+
+      # starts the client. 
+      def connect
+        begin
+          connection = AMQP.start((Workling.config[:amqp_options] ||{}).symbolize_keys)
+          @amq = MQ.new connection
+        rescue
+          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+        end
+      end
+
+      # no need for explicit closing. when the event loop
+      # terminates, the connection is closed anyway. 
+      def close; true; end
+
+      # subscribe to a queue
+      def subscribe(key)
+        @amq.queue(queue_for(key)).subscribe do |value|
+          data = Marshal.load(value) rescue value
+          yield data
+        end
+      end
+
+      # request and retrieve work
+      def retrieve(key); @amq.queue(queue_for(key)); end
+      def request(key, value); @amq.queue(queue_for(key)).publish(Marshal.dump(value)); end
+
+      private
+        def queue_for(key)
+          "#{Workling.config[:prefix]}#{key}"
+        end
+    end
+  end
+end

data/lib/workling/clients/amqp_exchange_client.rb

@@ -0,0 +1,58 @@
+#
+#  An Ampq client
+#
+module Workling
+  module Clients
+    class AmqpExchangeClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          require 'mq'
+        rescue Exception => e
+          raise WorklingError.new(
+            "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+            "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+          )
+        end
+      end
+
+      # starts the client. 
+      def connect
+        begin
+          @amq = MQ.new
+        rescue
+          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+        end
+      end
+
+      # no need for explicit closing. when the event loop
+      # terminates, the connection is closed anyway. 
+      def close; true; end
+
+      # subscribe to a queue
+      def subscribe(key)
+        @amq.queue(key).subscribe do |header, body|
+          value = YAML.load(body)
+          yield value
+        end
+      end
+
+      # request and retrieve work
+      def retrieve(key)
+          @amq.queue(key)
+      end
+
+      # publish message to exchange
+      # using the specified routing key
+      def request(exchange_name, value)
+        exchange_name = "amq.topic"
+
+        key = value.delete(:routing_key)
+        msg = YAML.dump(value)
+        exchange = @amq.topic(exchange_name)
+        exchange.publish(msg, :routing_key => key)
+      end
+
+    end
+  end
+end

data/lib/workling/clients/backgroundjob_client.rb

@@ -0,0 +1,25 @@
+module Workling
+  module Clients
+    class BackgroundjobClient < Workling::Clients::Base
+
+      def self.installed?
+        Object.const_defined? "Bj"
+      end
+
+      #  passes the job to bj by serializing the options to xml and passing them to
+      #  ./script/bj_invoker.rb, which in turn routes the deserialized args to the
+      #  appropriate worker. 
+      def dispatch(clazz, method, options = {})
+        stdin = Workling::Remote.routing.queue_for(clazz, method) + 
+                " " + 
+                options.to_xml(:indent => 0, :skip_instruct => true)
+                
+        Bj.submit "./script/runner ./script/bj_invoker.rb", 
+          :stdin => stdin
+        
+        return nil # that means nothing!
+      end
+
+    end
+  end
+end

data/lib/workling/clients/base.rb

@@ -0,0 +1,89 @@
+#
+#  Clients are responsible for dispatching jobs either to a broker (starling, rabbitmq etc) or invoke them (spawn)
+#
+#  Clients that involve a broker should subclass Workling::Clients::BrokerBase
+#
+#  Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
+#  from a specific type of job. 
+#
+module Workling
+  module Clients
+    class Base
+
+      #
+      # Load the required libraries, for this client
+      #
+      def self.load
+        
+      end
+
+
+      #
+      # See if the libraries required for this client are installed
+      #
+      def self.installed?
+        true
+      end
+
+
+      # returns the Workling::Base.logger
+      def logger; Workling::Base.logger; end
+
+
+      #
+      # Dispatch a job to the client. If this client uses a job broker, then
+      # this method should submit it, otherwise it should run the job
+      #
+      #    clazz: Name of the worker class
+      #    method: Name of the methods on the worker
+      #    options: optional arguments for the job
+      #
+      def dispatch(clazz, method, options = {})
+        raise NotImplementedError.new("Implement dispatch(clazz, method, options) in your client. ")
+      end
+
+
+      #
+      #  Requests a job on the broker.
+      #
+      #      work_type: 
+      #      arguments: the argument to the worker method
+      #
+      def request(work_type, arguments)
+        raise WorklingError.new("This client does not involve a broker.")
+      end
+
+      #
+      #  Gets job results off a job broker. Returns nil if there are no results. 
+      #
+      #      worker_uid: the uid returned by workling when the work was dispatched
+      #
+      def retrieve(work_uid)
+        raise WorklingError.new("This client does not involve a broker.")
+      end
+      
+      #
+      #  Subscribe to job results in a job broker.
+      #
+      #      worker_type: 
+      #
+      def subscribe(work_type)
+        raise WorklingError.new("This client does not involve a broker.")
+      end
+      
+      #
+      #  Opens a connection to the job broker.
+      #
+      def connect
+        raise WorklingError.new("This client does not involve a broker.")
+      end
+      
+      #
+      #  Closes the connection to the job broker. 
+      #
+      def close
+        raise WorklingError.new("This client does not involve a broker.")
+      end
+    end
+  end
+end

data/lib/workling/clients/broker_base.rb

@@ -0,0 +1,63 @@
+module Workling
+  module Clients
+    class BrokerBase < Base
+
+      #
+      # Dispatch a job to the client. If this client uses a job broker, then
+      # this method should submit it, otherwise it should run the job
+      #
+      #    clazz: Name of the worker class
+      #    method: Name of the methods on the worker
+      #    options: optional arguments for the job
+      #
+      def dispatch(clazz, method, options = {})
+        @connected ||= connect
+        request(Workling::Remote.routing.queue_for(clazz, method), options)
+        return nil
+      end
+
+
+      #
+      #  Requests a job on the broker.
+      #
+      #      work_type: 
+      #      arguments: the argument to the worker method
+      #
+      def request(work_type, arguments)
+        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
+      end    
+
+      #
+      #  Gets job results off a job broker. Returns nil if there are no results. 
+      #
+      #      worker_uid: the uid returned by workling when the work was dispatched
+      #
+      def retrieve(work_uid)
+        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+      end
+      
+      #
+      #  Subscribe to job results in a job broker.
+      #
+      #      worker_type: 
+      #
+      def subscribe(work_type)
+        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+      end
+      
+      #
+      #  Opens a connection to the job broker.
+      #
+      def connect
+        raise NotImplementedError.new("Implement connect() in your client. ")
+      end
+      
+      #
+      #  Closes the connection to the job broker. 
+      #
+      def close
+        raise NotImplementedError.new("Implement close() in your client. ")
+      end
+    end
+  end
+end

data/lib/workling/clients/memcache_queue_client.rb

@@ -0,0 +1,104 @@
+#
+#  This client can be used for all Queue Servers that speak Memcached, such as Starling. 
+#
+#  Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client, 
+#  or memcache-client, if this is not available. See the README for a discussion of the memcache 
+#  clients. 
+#
+#  method_missing delegates all messages through to the underlying memcache connection. 
+#
+module Workling
+  module Clients
+    class MemcacheQueueClient < Workling::Clients::BrokerBase
+
+      def self.installed?
+        begin
+          require 'starling' 
+        rescue LoadError
+        end
+
+        Object.const_defined? "Starling"
+      end
+
+      def self.load
+        begin
+          gem 'memcache-client'
+          require 'memcache'
+        rescue Gem::LoadError
+          Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
+        end
+      end
+
+      # the url with which the memcache client expects to reach starling
+      attr_accessor :queueserver_urls
+      
+      # the memcache connection object
+      attr_accessor :connection
+      
+      #
+      #  the client attempts to connect to queueserver using the configuration options found in 
+      #
+      #      Workling.config. this can be configured in config/workling.yml. 
+      #
+      #  the initialization code will raise an exception if memcache-client cannot connect 
+      #  to queueserver.
+      #
+      def connect
+        listens_on = Workling.config[:listens_on] || "localhost:22122"
+        @queueserver_urls = listens_on.split(',').map { |url| url ? url.strip : url }
+        options = [@queueserver_urls, Workling.config[:memcache_options]].compact
+        self.connection = ::MemCache.new(*options)
+
+        raise_unless_connected!
+      end
+
+      # closes the memcache connection
+      def close
+        self.connection.flush_all
+        self.connection.reset
+      end
+
+      # implements the client job request and retrieval 
+      def request(key, value)
+        set(key, value)
+      end
+      
+      def retrieve(key)
+        begin
+          get(key)
+        rescue MemCache::MemCacheError => e
+          # failed to enqueue, raise a workling error so that it propagates upwards
+          raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
+        end
+      end
+            
+      private
+        # make sure we can actually connect to queueserver on the given port
+        def raise_unless_connected!
+          begin 
+            self.connection.stats
+          rescue
+            raise Workling::QueueserverNotFoundError.new
+          end
+        end
+        
+        [:get, :set].each do |method|
+          class_eval <<-EOS
+          def #{method}(*args, &block)
+            self.connection.#{method}(*args, &block)
+          end
+          EOS
+        end
+
+        # delegates directly through to the memcache connection. 
+        def method_missing(method, *args)
+          begin
+            self.connection.send(method, *args)
+          rescue MemCache::MemCacheError => e
+            raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
+          end
+        end
+
+    end
+  end
+end

data/lib/workling/clients/memory_queue_client.rb

@@ -0,0 +1,34 @@
+module Workling
+  module Clients
+    class MemoryQueueClient < Workling::Clients::BrokerBase
+
+      def initialize
+        @subscribers ||= {}
+        @queues ||= {}
+      end
+
+      # collects the worker blocks in a hash
+      def subscribe(work_type, &block)
+        @subscribers[work_type] = block
+      end
+
+      # immediately invokes the required worker block
+      def request(work_type, arguments)
+        if subscription = @subscribers[work_type]
+          subscription.call(arguments)
+        else
+          @queues[work_type] ||= []
+          @queues[work_type] << arguments
+        end
+      end
+
+      def retrieve(work_type)
+        queue = @queues[work_type]
+        queue.pop if queue
+      end
+
+      def connect; true; end
+      def close; true; end
+    end
+  end
+end

data/lib/workling/clients/not_client.rb

@@ -0,0 +1,14 @@
+#
+#  Dont actually run the job...
+#
+module Workling
+  module Clients
+    class NotClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        return nil # nada. niente.
+      end
+
+    end
+  end
+end