derfred-workling 0.4.9.1 → 0.4.9.2

data/lib/cattr_accessor.rb

@@ -0,0 +1,51 @@
+# Extends the class object with class and instance accessors for class attributes,
+# just like the native attr* accessors for instance attributes.
+#
+#  class Person
+#    cattr_accessor :hair_colors
+#  end
+#
+#  Person.hair_colors = [:brown, :black, :blonde, :red]
+class Class
+  def cattr_reader(*syms)
+    syms.flatten.each do |sym|
+      next if sym.is_a?(Hash)
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}  # unless defined? @@hair_colors
+          @@#{sym} = nil          #   @@hair_colors = nil
+        end                       # end
+                                  #
+        def self.#{sym}           # def self.hair_colors
+          @@#{sym}                #   @@hair_colors
+        end                       # end
+                                  #
+        def #{sym}                # def hair_colors
+          @@#{sym}                #   @@hair_colors
+        end                       # end
+      EOS
+    end
+  end
+
+  def cattr_writer(*syms)
+    syms.flatten.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}                       # unless defined? @@hair_colors
+          @@#{sym} = nil                               #   @@hair_colors = nil
+        end                                            # end
+                                                       #
+        def self.#{sym}=(obj)                          # def self.hair_colors=(obj)
+          @@#{sym} = obj                               #   @@hair_colors = obj
+        end                                            # end
+                                                       #
+        def #{sym}=(obj)                               # def hair_colors=(obj)
+          @@#{sym} = obj                               #   @@hair_colors = obj
+        end                                            # end
+      EOS
+    end
+  end
+
+  def cattr_accessor(*syms)
+    cattr_reader(*syms)
+    cattr_writer(*syms)
+  end
+end

data/lib/mattr_accessor.rb

@@ -0,0 +1,55 @@
+# Extends the module object with module and instance accessors for class attributes, 
+# just like the native attr* accessors for instance attributes.
+#
+#  module AppConfiguration
+#    mattr_accessor :google_api_key
+#    self.google_api_key = "123456789"
+#
+#    mattr_accessor :paypal_url
+#    self.paypal_url = "www.sandbox.paypal.com"
+#  end
+#
+#  AppConfiguration.google_api_key = "overriding the api key!"
+class Module
+  def mattr_reader(*syms)
+    syms.each do |sym|
+      next if sym.is_a?(Hash)
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}  # unless defined? @@pagination_options
+          @@#{sym} = nil          #   @@pagination_options = nil
+        end                       # end
+                                  #
+        def self.#{sym}           # def self.pagination_options
+          @@#{sym}                #   @@pagination_options
+        end                       # end
+                                  #
+        def #{sym}                # def pagination_options
+          @@#{sym}                #   @@pagination_options
+        end                       # end
+      EOS
+    end
+  end
+  
+  def mattr_writer(*syms)
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}                       # unless defined? @@pagination_options
+          @@#{sym} = nil                               #   @@pagination_options = nil
+        end                                            # end
+                                                       #
+        def self.#{sym}=(obj)                          # def self.pagination_options=(obj)
+          @@#{sym} = obj                               #   @@pagination_options = obj
+        end                                            # end
+                                                       #
+        def #{sym}=(obj)                               # def pagination_options=(obj)
+          @@#{sym} = obj                               #   @@pagination_options = obj
+        end                                            # end
+      EOS
+    end
+  end
+  
+  def mattr_accessor(*syms)
+    mattr_reader(*syms)
+    mattr_writer(*syms)
+  end
+end

data/lib/rude_q/client.rb

@@ -0,0 +1,11 @@
+#
+#  A RudeQ client that behvaes somewhat like memcache-client
+#
+module RudeQ
+  class Client    
+    def initialize(*args); super(); end   
+    def set(key, value); RudeQueue.set(key, value); end;
+    def get(key); RudeQueue.get(key); end;
+    def stats; ActiveRecord::Base.connection; end
+  end
+end

data/lib/workling/base.rb

@@ -0,0 +1,115 @@
+#require 'struct'
+
+#
+#  All worker classes must inherit from this class, and be saved in app/workers. 
+# 
+#  The Worker lifecycle: 
+#    The Worker is loaded once, at which point the instance method 'create' is called. 
+#
+#  Invoking Workers: 
+#    Calling async_my_method on the worker class will trigger background work.
+#    This means that the loaded Worker instance will receive a call to the method
+#    my_method(:uid => "thisjobsuid2348732947923"). 
+#
+#    The Worker method must have a single hash argument. Note that the job :uid will
+#    be merged into the hash. 
+#
+module Workling
+  class Base
+    cattr_writer :logger
+    def self.logger
+      @@logger ||= ::RAILS_DEFAULT_LOGGER
+    end
+
+    cattr_accessor :exposed_methods
+    @@exposed_methods ||= {}
+
+    def self.inherited(subclass)
+      Workling::Discovery.discovered << subclass
+    end
+
+    # expose a method using a custom queue name
+    def self.expose(method, options)
+      self.exposed_methods[method.to_s] = options[:as]
+    end
+
+    # identify the queue for a given method
+    def self.queue_for(method)
+      if self.exposed_methods.has_key?(method)
+        self.exposed_methods[method]
+      else
+        "#{ self.to_s.tableize }/#{ method }".split("/").join("__") # Don't split with : because it messes up memcache stats
+      end
+    end
+
+    # identify the method linked to the queue
+    def self.method_for(queue)
+      if self.exposed_methods.invert.has_key?(queue)
+        self.exposed_methods.invert[queue]
+      else
+        queue.split("__").last
+      end
+    end
+
+    def method_for(queue)
+      self.class.method_for(queue)
+    end
+
+
+    def initialize
+      super
+      
+      create
+    end
+
+    # Put worker initialization code in here. This is good for restarting jobs that
+    # were interrupted.
+    def create
+    end
+
+    # override this if you want to set up exception notification
+    def notify_exception(exception, method, options)
+      logger.error "WORKLING ERROR: runner could not invoke #{ self.class }:#{ method } with #{ options.inspect }. error was: #{ exception.inspect }\n #{ exception.backtrace.join("\n") }"
+    end
+
+    # takes care of suppressing remote errors but raising Workling::WorklingNotFoundError
+    # where appropriate. swallow workling exceptions so that everything behaves like remote code.
+    # otherwise StarlingRunner and SpawnRunner would behave too differently to NotRemoteRunner.
+    def dispatch_to_worker_method(method, options = {})
+      begin
+        options = default_options.merge(options)
+        self.send(method, options)
+      rescue Workling::WorklingError => e
+        raise e
+      rescue Exception => e
+        notify_exception e, method, options
+        on_error(e) if respond_to?(:on_error)
+
+        # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
+        raise e if Workling.raise_exceptions?
+      end
+    end    
+
+    # supply default_options as a hash in classes that inherit Workling::Base
+    def default_options
+      {}
+    end
+  
+    # thanks to blaine cook for this suggestion.
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^asynch?_(.*)/
+        Workling::Remote.run(self.to_s.dasherize, $1, *args)
+      else
+        super
+      end
+    end
+    
+    def self.logger
+      @logger ||= defined?(RAILS_DEFAULT_LOGGER) ? ::RAILS_DEFAULT_LOGGER : Logger.new($stdout)
+    end
+
+    def logger
+      self.class.logger
+    end
+  end
+end

data/lib/workling/clients/amqp_client.rb

@@ -0,0 +1,38 @@
+require 'workling/clients/base'
+Workling.try_load_an_amqp_client
+
+#
+#  An Ampq client
+#
+module Workling
+  module Clients
+    class AmqpClient < Workling::Clients::Base
+      
+      # starts the client. 
+      def connect
+        begin
+          connection = AMQP.start((Workling.config[:amqp_options] ||{}).symbolize_keys)
+          @amq = MQ.new connection
+        rescue
+          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+        end
+      end
+      
+      # no need for explicit closing. when the event loop
+      # terminates, the connection is closed anyway. 
+      def close; true; end
+      
+      # subscribe to a queue
+      def subscribe(key)
+        @amq.queue(key).subscribe do |value|
+          data = Marshal.load(value) rescue value
+          yield data
+        end
+      end
+      
+      # request and retrieve work
+      def retrieve(key); @amq.queue(key); end
+      def request(key, value); @amq.queue(key).publish(Marshal.dump(value)); end
+    end
+  end
+end

data/lib/workling/clients/amqp_exchange_client.rb

@@ -0,0 +1,50 @@
+require 'workling/clients/base'
+Workling.try_load_an_amqp_client
+
+#
+#  An Ampq client
+#
+module Workling
+  module Clients
+    class AmqpExchangeClient < Workling::Clients::Base
+      
+      # starts the client. 
+      def connect
+        begin
+          @amq = MQ.new
+        rescue
+          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
+        end
+      end
+      
+      # no need for explicit closing. when the event loop
+      # terminates, the connection is closed anyway. 
+      def close; true; end
+      
+      # subscribe to a queue
+      def subscribe(key)
+        @amq.queue(key).subscribe do |header, body|
+
+          puts "***** received msg with header - #{header.inspect}"
+
+          value = YAML.load(body)
+          yield value
+        end
+      end
+      
+      # request and retrieve work
+      def retrieve(key)
+          @amq.queue(key)
+      end
+
+      # publish message to exchange
+      # using the specified routing key
+      def request(exchange_name, value)
+        key = value.delete(:routing_key)
+        msg = YAML.dump(value)
+        exchange = @amq.topic(exchange_name)
+        exchange.publish(msg, :routing_key => key)
+      end
+    end
+  end
+end

data/lib/workling/clients/base.rb

@@ -0,0 +1,57 @@
+#
+#  Clients are responsible for communicating with a job broker (ie connecting to starling or rabbitmq.)
+#
+#  Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
+#  from a specific type of job. 
+#
+module Workling
+  module Clients
+    class Base
+
+      # returns the Workling::Base.logger
+      def logger; Workling::Base.logger; end
+
+      #
+      #  Requests a job on the broker.
+      #
+      #      work_type: 
+      #      arguments: the argument to the worker method
+      #
+      def request(work_type, arguments)
+        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
+      end    
+
+      #
+      #  Gets job results off a job broker. Returns nil if there are no results. 
+      #
+      #      worker_uid: the uid returned by workling when the work was dispatched
+      #
+      def retrieve(work_uid)
+        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
+      end
+      
+      #
+      #  Subscribe to job results in a job broker.
+      #
+      #      worker_type: 
+      #
+      def subscribe(work_type)
+        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
+      end
+      
+      #
+      #  Opens a connection to the job broker.
+      #
+      def connect
+        raise NotImplementedError.new("Implement connect() in your client. ")
+      end
+      
+      #
+      #  Closes the connection to the job broker. 
+      #
+      def close
+        raise NotImplementedError.new("Implement close() in your client. ")
+      end
+    end
+  end
+end

data/lib/workling/clients/memcache_queue_client.rb

@@ -0,0 +1,91 @@
+require 'workling/clients/base'
+require 'memcache'
+
+#
+#  This client can be used for all Queue Servers that speak Memcached, such as Starling. 
+#
+#  Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client, 
+#  or memcache-client, if this is not available. See the README for a discussion of the memcache 
+#  clients. 
+#
+#  method_missing delegates all messages through to the underlying memcache connection. 
+#
+module Workling
+  module Clients
+    class MemcacheQueueClient < Workling::Clients::Base
+      
+      # the class with which the connection is instantiated
+      cattr_accessor :memcache_client_class
+      @@memcache_client_class ||= ::MemCache
+      
+      # the url with which the memcache client expects to reach starling
+      attr_accessor :queueserver_urls
+      
+      # the memcache connection object
+      attr_accessor :connection
+      
+      #
+      #  the client attempts to connect to queueserver using the configuration options found in 
+      #
+      #      Workling.config. this can be configured in config/workling.yml. 
+      #
+      #  the initialization code will raise an exception if memcache-client cannot connect 
+      #  to queueserver.
+      #
+      def connect
+        @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url }
+        options = [@queueserver_urls, Workling.config[:memcache_options]].compact
+        self.connection = MemcacheQueueClient.memcache_client_class.new(*options)
+        
+        raise_unless_connected!
+      end
+      
+      # closes the memcache connection
+      def close
+        self.connection.flush_all
+        self.connection.reset
+      end
+
+      # implements the client job request and retrieval 
+      def request(key, value)
+        set(key, value)
+      end
+      
+      def retrieve(key)
+        begin
+          get(key)
+        rescue MemCache::MemCacheError => e
+          # failed to enqueue, raise a workling error so that it propagates upwards
+          raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
+        end
+      end
+            
+      private
+        # make sure we can actually connect to queueserver on the given port
+        def raise_unless_connected!
+          begin 
+            self.connection.stats
+          rescue
+            raise Workling::QueueserverNotFoundError.new
+          end
+        end
+        
+        [:get, :set].each do |method|
+          class_eval <<-EOS
+          def #{method}(*args, &block)
+            self.connection.#{method}(*args, &block)
+          end
+          EOS
+        end
+
+        # delegates directly through to the memcache connection. 
+        def method_missing(method, *args)
+          begin
+            self.connection.send(method, *args)
+          rescue MemCache::MemCacheError => e
+            raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
+          end
+        end
+    end
+  end
+end

data/lib/workling/clients/sqs_client.rb

@@ -0,0 +1,162 @@
+require 'workling/clients/base'
+require 'json'
+require 'right_aws'
+
+#
+#  An SQS client
+#
+# Requires the following configuration in workling.yml:
+#
+# production:
+#   sqs_options:
+#     aws_access_key_id: <your AWS access key id>
+#     aws_secret_access_key: <your AWS secret access key>
+#
+# You can also add the following optional parameters:
+#
+#     # Queue names consist of an optional prefix, followed by the environment
+#     # and the name of the key.
+#     prefix: foo_
+#
+#     # The number of SQS messages to retrieve at once. The maximum and default
+#     # value is 10.
+#     messages_per_req: 5
+#
+#     # The SQS visibility timeout for retrieved messages. Defaults to 30 seconds.
+#     visibility_timeout: 15
+#   
+module Workling
+  module Clients
+    class SqsClient < Workling::Clients::Base
+
+      AWS_MAX_QUEUE_NAME = 80
+
+      # Note that 10 is the maximum number of messages that can be retrieved
+      # in a single request.
+      DEFAULT_MESSAGES_PER_REQ = 10
+      DEFAULT_VISIBILITY_TIMEOUT = 30
+      DEFAULT_VISIBILITY_RESERVE = 10
+      
+      # Mainly exposed for testing purposes
+      attr_reader :sqs_options
+      attr_reader :messages_per_req
+      attr_reader :visibility_timeout
+      
+      # Starts the client. 
+      def connect
+        @sqs_options = Workling.config[:sqs_options]
+
+        # Make sure that required options were specified
+        unless (@sqs_options.include?('aws_access_key_id') &&
+               @sqs_options.include?('aws_secret_access_key'))
+          raise WorklingError, 'Unable to start SqsClient due to missing SQS options'
+        end
+        
+        # Optional settings
+        @messages_per_req = @sqs_options['messages_per_req'] || DEFAULT_MESSAGES_PER_REQ
+        @visibility_timeout = @sqs_options['visibility_timeout'] || DEFAULT_VISIBILITY_TIMEOUT
+        @visibility_reserve = @sqs_options['visibility_reserve'] || DEFAULT_VISIBILITY_RESERVE
+        
+        begin
+          @sqs = RightAws::SqsGen2.new(
+            @sqs_options['aws_access_key_id'],
+            @sqs_options['aws_secret_access_key'],
+            :multi_thread => true)
+        rescue => e
+          raise WorklingError, "Unable to connect to SQS. Error: #{e}"
+        end
+      end
+      
+      # No need for explicit closing, since there is no persistent
+      # connection to SQS.
+      def close
+        true
+      end
+      
+      # Retrieve work.
+      def retrieve(key)
+        begin
+          # We're using a buffer per key to retrieve several messages at once,
+          # then return them one at a time until the buffer is empty.
+          # Workling seems to create one thread per worker class, each with its own
+          # client. But to be sure (and to be less dependent on workling internals),
+          # we store each buffer in a thread local variable.
+          buffer = Thread.current["buffer_#{key}"]
+          if buffer.nil? || buffer.empty?
+            Thread.current["buffer_#{key}"] = buffer = queue_for_key(key).receive_messages(
+              @messages_per_req, @visibility_timeout)
+          end
+
+          if buffer.empty?
+            nil
+          else
+            msg = buffer.shift
+
+            # We need to protect against the case that processing one of the
+            # messages in the buffer took so much time that the visibility
+            # timeout for the remaining messages has expired. To be on the
+            # safe side (since we need to leave enough time to delete the
+            # message), we drop it if more than half of the visibility timeout
+            # has elapsed.
+            if msg.received_at < (Time.now - (@visibility_timeout - @visibility_reserve))
+              nil
+            else
+              # Need to wrap in HashWithIndifferentAccess, as JSON serialization
+              # loses symbol keys.
+              parsed_msg = HashWithIndifferentAccess.new(JSON.parse(msg.body))
+            
+              # Delete the msg from SQS, so we don't re-retrieve it after the
+              # visibility timeout. Ideally we would defer deleting a msg until
+              # after Workling has successfully processed it, but it currently
+              # doesn't provide the necessary hooks for this.
+              msg.delete
+            
+              parsed_msg
+            end
+          end
+
+        rescue => e
+          logger.error "Error retrieving msg for key: #{key}; Error: #{e}\n#{e.backtrace.join("\n")}"
+        end
+        
+      end
+
+      # Request work.
+      def request(key, value)
+        begin
+          queue_for_key(key).send_message(value.to_json)
+        rescue => e
+          logger.error "SQS Client: Error sending msg for key: #{key}, value: #{value.inspect}; Error: #{e}"
+          raise WorklingError, "Error sending msg for key: #{key}, value: #{value.inspect}; Error: #{e}"
+        end
+      end
+      
+      # Returns the queue that corresponds to the specified key. Creates the
+      # queue if it doesn't exist yet.
+      def queue_for_key(key)
+        # Use thread local for storing queues, for the same reason as for buffers
+        Thread.current["queue_#{key}"] ||= @sqs.queue(queue_name(key), true, @visibility_timeout)
+      end
+      
+      # Returns the queue name for the specified key. The name consists of an
+      # optional prefix, followed by the environment and the key itself. Note
+      # that with a long worker class / method name, the name could exceed the
+      # 80 character maximum for SQS queue names. We truncate the name until it
+      # fits, but there's still the danger of this not being unique any more.
+      # Might need to implement a more robust naming scheme...
+      def queue_name(key)
+        "#{@sqs_options['prefix'] || ''}#{env}_#{key}"[0, AWS_MAX_QUEUE_NAME]
+      end
+      
+      private
+      
+      def logger
+        Rails.logger
+      end
+      
+      def env
+        Rails.env
+      end
+    end
+  end
+end