workling 0.4.9.7

data/lib/workling/clients/not_remote_client.rb

@@ -0,0 +1,17 @@
+#
+#  Run the job inline
+#
+module Workling
+  module Clients
+    class NotRemoteClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        options = Marshal.load(Marshal.dump(options)) # get this to behave more like the remote runners
+        Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+
+        return nil # nada. niente.
+      end
+
+    end
+  end
+end

data/lib/workling/clients/rude_q_client.rb

@@ -0,0 +1,47 @@
+#
+#  This client interfaces with RudeQ a databased queue system
+#   http://github.com/matthewrudy/rudeq
+#
+module Workling
+  module Clients
+    class RudeQClient < Workling::Clients::BrokerBase
+
+      def self.installed?
+        begin
+          gem 'rudeq'
+          require 'rudeq' 
+        rescue LoadError
+        end
+
+        Object.const_defined? "RudeQueue"
+      end
+
+      def self.load
+        begin
+          gem 'rudeq'
+          require 'rudeq'
+        rescue Gem::LoadError
+          Workling::Base.logger.info "WORKLING: couldn't find rudeq library. Install: \"gem install matthewrudy-rudeq\". "
+        end
+      end
+
+      # no-op as the db connection should exists always
+      def connect
+      end
+
+      # again a no-op as we would want to yank out the db connection behind the apps back
+      def close
+      end
+
+      # implements the client job request and retrieval 
+      def request(key, value)
+        RudeQueue.set(key, value)
+      end
+
+      def retrieve(key)
+        RudeQueue.get(key)
+      end
+
+    end
+  end
+end

data/lib/workling/clients/spawn_client.rb

@@ -0,0 +1,46 @@
+#
+#  Run the job over the spawn plugin. Refer to the README for instructions on 
+#  installing Spawn. 
+#
+#  Spawn forks the entire process once for each job. This means that the job starts 
+#  with a very low latency, but takes up more memory for each job. 
+# 
+#  It's also possible to configure Spawn to start a Thread for each job. Do this
+#  by setting
+#
+#      Workling::Clients::SpawnClient.options = { :method => :thread }
+#
+#  Have a look at the Spawn README to find out more about the characteristics of this. 
+#
+module Workling
+  module Clients
+    class SpawnClient < Workling::Clients::Base
+
+      def self.installed?
+        begin
+          require 'spawn'
+        rescue LoadError
+        end
+
+        Object.const_defined? "Spawn"
+      end
+
+      cattr_writer :options
+      def self.options
+        # use thread for development and test modes. easier to hunt down exceptions that way.
+        @@options ||= { :method => (RAILS_ENV == "test" || RAILS_ENV == "development" ? :fork : :thread) }
+      end
+
+      include Spawn if installed?
+
+      def dispatch(clazz, method, options = {})
+        spawn(SpawnClient.options) do # exceptions are trapped in here. 
+          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+        end
+
+        return nil # that means nothing!
+      end
+
+    end
+  end
+end

data/lib/workling/clients/sqs_client.rb

@@ -0,0 +1,163 @@
+require 'json'
+require 'right_aws'
+
+#
+#  An SQS client
+#
+# Requires the following configuration in workling.yml:
+#
+# production:
+#   sqs_options:
+#     aws_access_key_id: <your AWS access key id>
+#     aws_secret_access_key: <your AWS secret access key>
+#
+# You can also add the following optional parameters:
+#
+#     # Queue names consist of an optional prefix, followed by the environment
+#     # and the name of the key.
+#     prefix: foo_
+#
+#     # The number of SQS messages to retrieve at once. The maximum and default
+#     # value is 10.
+#     messages_per_req: 5
+#
+#     # The SQS visibility timeout for retrieved messages. Defaults to 30 seconds.
+#     visibility_timeout: 15
+#   
+module Workling
+  module Clients
+    class SqsClient < Workling::Clients::BrokerBase
+
+      unless defined?(AWS_MAX_QUEUE_NAME)
+        AWS_MAX_QUEUE_NAME = 80
+
+        # Note that 10 is the maximum number of messages that can be retrieved
+        # in a single request.
+        DEFAULT_MESSAGES_PER_REQ = 10
+        DEFAULT_VISIBILITY_TIMEOUT = 30
+        DEFAULT_VISIBILITY_RESERVE = 10
+      end
+      
+      # Mainly exposed for testing purposes
+      attr_reader :sqs_options
+      attr_reader :messages_per_req
+      attr_reader :visibility_timeout
+      
+      # Starts the client. 
+      def connect
+        @sqs_options = Workling.config[:sqs_options]
+
+        # Make sure that required options were specified
+        unless (@sqs_options.include?('aws_access_key_id') &&
+               @sqs_options.include?('aws_secret_access_key'))
+          raise WorklingError, 'Unable to start SqsClient due to missing SQS options'
+        end
+        
+        # Optional settings
+        @messages_per_req = @sqs_options['messages_per_req'] || DEFAULT_MESSAGES_PER_REQ
+        @visibility_timeout = @sqs_options['visibility_timeout'] || DEFAULT_VISIBILITY_TIMEOUT
+        @visibility_reserve = @sqs_options['visibility_reserve'] || DEFAULT_VISIBILITY_RESERVE
+        
+        begin
+          @sqs = RightAws::SqsGen2.new(
+            @sqs_options['aws_access_key_id'],
+            @sqs_options['aws_secret_access_key'],
+            :multi_thread => true)
+        rescue => e
+          raise WorklingError, "Unable to connect to SQS. Error: #{e}"
+        end
+      end
+      
+      # No need for explicit closing, since there is no persistent
+      # connection to SQS.
+      def close
+        true
+      end
+      
+      # Retrieve work.
+      def retrieve(key)
+        begin
+          # We're using a buffer per key to retrieve several messages at once,
+          # then return them one at a time until the buffer is empty.
+          # Workling seems to create one thread per worker class, each with its own
+          # client. But to be sure (and to be less dependent on workling internals),
+          # we store each buffer in a thread local variable.
+          buffer = Thread.current["buffer_#{key}"]
+          if buffer.nil? || buffer.empty?
+            Thread.current["buffer_#{key}"] = buffer = queue_for_key(key).receive_messages(
+              @messages_per_req, @visibility_timeout)
+          end
+
+          if buffer.empty?
+            nil
+          else
+            msg = buffer.shift
+
+            # We need to protect against the case that processing one of the
+            # messages in the buffer took so much time that the visibility
+            # timeout for the remaining messages has expired. To be on the
+            # safe side (since we need to leave enough time to delete the
+            # message), we drop it if more than half of the visibility timeout
+            # has elapsed.
+            if msg.received_at < (Time.now - (@visibility_timeout - @visibility_reserve))
+              nil
+            else
+              # Need to wrap in HashWithIndifferentAccess, as JSON serialization
+              # loses symbol keys.
+              parsed_msg = HashWithIndifferentAccess.new(JSON.parse(msg.body))
+            
+              # Delete the msg from SQS, so we don't re-retrieve it after the
+              # visibility timeout. Ideally we would defer deleting a msg until
+              # after Workling has successfully processed it, but it currently
+              # doesn't provide the necessary hooks for this.
+              msg.delete
+            
+              parsed_msg
+            end
+          end
+
+        rescue => e
+          logger.error "Error retrieving msg for key: #{key}; Error: #{e}\n#{e.backtrace.join("\n")}"
+        end
+        
+      end
+
+      # Request work.
+      def request(key, value)
+        begin
+          queue_for_key(key).send_message(value.to_json)
+        rescue => e
+          logger.error "SQS Client: Error sending msg for key: #{key}, value: #{value.inspect}; Error: #{e}"
+          raise WorklingError, "Error sending msg for key: #{key}, value: #{value.inspect}; Error: #{e}"
+        end
+      end
+      
+      # Returns the queue that corresponds to the specified key. Creates the
+      # queue if it doesn't exist yet.
+      def queue_for_key(key)
+        # Use thread local for storing queues, for the same reason as for buffers
+        Thread.current["queue_#{key}"] ||= @sqs.queue(queue_name(key), true, @visibility_timeout)
+      end
+      
+      # Returns the queue name for the specified key. The name consists of an
+      # optional prefix, followed by the environment and the key itself. Note
+      # that with a long worker class / method name, the name could exceed the
+      # 80 character maximum for SQS queue names. We truncate the name until it
+      # fits, but there's still the danger of this not being unique any more.
+      # Might need to implement a more robust naming scheme...
+      def queue_name(key)
+        "#{@sqs_options['prefix'] || ''}#{env}_#{key}"[0, AWS_MAX_QUEUE_NAME]
+      end
+      
+      private
+      
+      def logger
+        Rails.logger
+      end
+      
+      def env
+        Rails.env
+      end
+    end
+  end
+end

data/lib/workling/clients/thread_client.rb

@@ -0,0 +1,18 @@
+#
+#  Spawns a Thread. Used for Tests only, to simulate a remote runner more realistically. 
+#
+module Workling
+  module Clients
+    class ThreadClient < Workling::Clients::Base
+
+      def dispatch(clazz, method, options = {})
+        Thread.new {
+          Workling.find(clazz, method).dispatch_to_worker_method(method, options)
+        }
+
+        return nil
+      end
+
+    end
+  end
+end

data/lib/workling/clients/xmpp_client.rb

@@ -0,0 +1,110 @@
+#
+#  An XMPP client
+#
+#  How to use: this client requires the xmpp4r gem
+#
+#  in the config/environments/development.rb file (or production.rb etc)
+#
+#    Workling::Remote::Runners::ClientRunner.client = Workling::Clients::XmppClient.new
+#    Workling::Remote.dispatcher = Workling::Remote::Runners::ClientRunner.new        # dont use the standard runner
+#    Workling::Remote.invoker = Workling::Remote::Invokers::LoopedSubscriber          # does not work with the EventmachineSubscriber Invoker
+#
+#  furthermore in the workling.yml file you need to set up the server details for your XMPP server
+#
+#    development:
+#      listens_on: "localhost:22122"
+#      jabber_id: "sub@localhost/laptop"
+#      jabber_server: "localhost"
+#      jabber_password: "sub"
+#      jabber_service: "pubsub.derfredtop.local"
+#
+#  for details on how to configure your XMPP server (ejabberd) check out the following howto:
+#
+#    http://keoko.wordpress.com/2008/12/17/xmpp-pubsub-with-ejabberd-and-xmpp4r/
+#
+#
+#  finally you need to expose your worker methods to XMPP nodes like so:
+# 
+#    class NotificationWorker < Workling::Base
+# 
+#      expose :receive_notification, :as => "/home/localhost/pub/sub"
+# 
+#      def receive_notification(input)
+#        # something here
+#      end
+# 
+#    end
+#
+
+
+
+module Workling
+  module Clients
+    class XmppClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          gem "xmpp4r"
+          require 'xmpp4r'
+          require "xmpp4r/pubsub"
+          require "xmpp4r/pubsub/helper/servicehelper.rb"
+          require "xmpp4r/pubsub/helper/nodebrowser.rb"
+          require "xmpp4r/pubsub/helper/nodehelper.rb"
+        rescue Exception => e
+          raise WorklingError.new("Couldnt load the XMPP library. check that you have the xmpp4r gem installed")
+        end
+      end
+
+      # starts the client. 
+      def connect
+        begin
+          @client = Jabber::Client.new Workling.config[:jabber_id]
+          @client.connect Workling.config[:jabber_server]
+          @client.auth Workling.config[:jabber_password]
+          @client.send Jabber::Presence.new.set_type(:available)
+          @pubsub = Jabber::PubSub::ServiceHelper.new(@client, Workling.config[:jabber_service])
+          unsubscribe_from_all                # make sure there are no open subscriptions, could cause multiple delivery of notifications, as they are persistent
+        rescue
+          raise WorklingError.new("couldn't connect to the jabber server")
+        end
+      end
+
+      # disconnect from the server
+      def close
+        @client.close
+      end
+
+      # subscribe to a queue
+      def subscribe(key)
+        @pubsub.subscribe_to(key)
+
+        # filter out the subscription notification message that was generated by subscribing to the node
+        @pubsub.get_subscriptions_from_all_nodes()
+
+        @pubsub.add_event_callback do |event|
+          event.payload.each do |e|
+            e.children.each do |child|
+              yield Hash.from_xml(child.children.first.to_s) if child.name == 'item'
+            end
+          end
+        end
+      end
+
+      # request and retrieve work
+      def retrieve(key)
+        @pubsub.get_items_from(key, 1)
+      end
+
+      def request(key, value)
+        @pubsub.publish_item_to(key, value)
+      end
+
+      private
+        def unsubscribe_from_all
+          @pubsub.get_subscriptions_from_all_nodes.each do |subscription|
+            @pubsub.unsubscribe_from subscription.node
+          end
+        end
+    end
+  end
+end

data/lib/workling/discovery.rb

@@ -0,0 +1,16 @@
+#
+#  Discovery is responsible for loading workers in app/workers.
+#
+module Workling
+  class Discovery
+    cattr_accessor :discovered
+    @@discovered ||= []
+
+    # requires worklings so that they are added to routing.
+    def self.discover!
+      Workling.load_path.each do |p|
+        Dir.glob(p).each { |wling| require wling }
+      end
+    end
+  end
+end

data/lib/workling/invokers/amqp_single_subscriber.rb

@@ -0,0 +1,42 @@
+require 'eventmachine'
+
+#
+# TODO - Subscribes a single worker to a single queue 
+# 
+module Workling
+  module Invokers
+    class AmqpSingleSubscriber < Workling::Invokers::Base
+
+      def initialize(routing, client_class)
+        super
+      end
+
+      #
+      # Starts EM loop and sets up subscription callback for the worker
+      # Create the queue and bind to exchange using the routing key
+      #
+      def listen
+        EM.run do
+          connect do
+            queue_name = @routing.queue_for
+            routing_key = @routing.routing_key_for
+      
+            # temp stuff to hook the queues and exchanges up
+            # wildcard routing - # (match all)
+            exch = MQ.topic
+            q = MQ.queue(queue_name)
+            q.bind(exch, :key => routing_key)
+      
+            @client.subscribe(queue_name) do |args|
+              run(queue_name, args)
+            end
+          end
+        end
+      end
+
+      def stop
+        EM.stop if EM.reactor_running?
+      end
+    end
+  end
+end