workling 0.4.9.7

data/TODO.markdown

@@ -0,0 +1,27 @@
+# Todos for 0.5.0
+
+* add a linting runner for tests. should check that no ar objects are being passed around
+* add a mechanism for requiring models, for those people who insist on passing models across the wire
+* add reloading of workers if Rails.reload?
+* write some code that knows if the client should be started, and gives out a warning
+* add a configuration option for SERVER/CLIENT
+* add phusion daemon starter option so that workling_client doesn't need to be started manually on SERVER
+* write some more documentation on the above issues and on standard remote setup. 
+* create a public forum, rdoc site
+* try to reduce user error in setting environments correctly
+* add beanstalkd runner
+* refactor starling* to be memcache*. add aliased classes into deprecated.rb.
+* look into create method. is this being called more often than intended?
+* add some monit and god scripts as starters
+* try to catch more user setup errors which lead to worker code not being called
+* add json as a marshaling option for the amqp client. 
+
+# Todos for 1.0
+
+* gemify
+* move all runner/invoker implementations out of workling
+* move backend discovery code out of workling
+* decide on a single backend to include in workling
+* merb support
+* test on jruby
+* more runners: sqs

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:minor: 4
+:patch: 2
+:major: 0

data/bin/workling_client

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'daemons'
+require File.join(File.dirname(__FILE__), '../lib/workling_daemon')
+
+daemon_options = {
+  :app_name   => "workling",
+  :dir_mode   => :normal,
+  :dir        => File.join(Dir.pwd, 'log'),
+  :log_output => true,
+  :multiple   => false,
+  :backtrace  => true,
+  :monitor    => false
+}.merge(WorklingDaemon.parse_daemon_options(ARGV))
+
+workling_options = {
+  :client_class => "memcache_queue",
+  :invoker_class => "threaded_poller",
+  :routing_class => "class_and_method",
+  :rails_root => Dir.pwd,
+  :load_path => ['app/workers/**/*.rb'],
+  :rails_env => (ENV['RAILS_ENV'] || "development").dup,
+  :no_rails => false
+}.merge(WorklingDaemon.parse_workling_options(ARGV))
+
+Daemons.run_proc(daemon_options[:app_name], daemon_options) do
+  Dir.chdir(workling_options[:rails_root])
+  WorklingDaemon.run(workling_options)
+end

data/contrib/bj_invoker.rb

@@ -0,0 +1,11 @@
+@routing = Workling::Routing::ClassAndMethodRouting.new
+unnormalized = REXML::Text::unnormalize(STDIN.read)
+message, command, args = *unnormalized.match(/(^[^ ]*) (.*)/)
+options = Hash.from_xml(args)["hash"]
+
+if workling = @routing[command]
+  options = options.symbolize_keys
+  method_name = @routing.method_name(command)
+
+  workling.dispatch_to_worker_method(method_name, options)
+end

data/contrib/starling_status.rb

@@ -0,0 +1,37 @@
+require 'pp'
+
+puts '=> Loading Rails...'
+
+require File.dirname(__FILE__) + '/../config/environment'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/remote/invokers/basic_poller'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/routing/class_and_method_routing'
+
+puts '** Rails loaded.'
+
+trap(:INT) { exit }
+
+client = Workling::Clients::MemcacheQueueClient.new
+  
+begin
+  client.connect
+  client.reset
+  
+  client.stats # do this so that connection is shown as established below. 
+  
+  puts "Queue state:"
+  pp client.inspect
+  pp "Active?: #{client.active?}"
+  pp "Read Only?: #{client.readonly?}"
+  puts ""
+  puts "Servers:"
+  pp client.servers
+  puts ""
+  puts "Queue stats:"
+  pp client.stats
+
+  puts "\nThread Stats:"
+  pp Thread.list
+ensure
+  puts '** Exiting'
+  client.close
+end

data/lib/extensions/cattr_accessor.rb

@@ -0,0 +1,51 @@
+# Extends the class object with class and instance accessors for class attributes,
+# just like the native attr* accessors for instance attributes.
+#
+#  class Person
+#    cattr_accessor :hair_colors
+#  end
+#
+#  Person.hair_colors = [:brown, :black, :blonde, :red]
+class Class
+  def cattr_reader(*syms)
+    syms.flatten.each do |sym|
+      next if sym.is_a?(Hash)
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}  # unless defined? @@hair_colors
+          @@#{sym} = nil          #   @@hair_colors = nil
+        end                       # end
+                                  #
+        def self.#{sym}           # def self.hair_colors
+          @@#{sym}                #   @@hair_colors
+        end                       # end
+                                  #
+        def #{sym}                # def hair_colors
+          @@#{sym}                #   @@hair_colors
+        end                       # end
+      EOS
+    end
+  end
+
+  def cattr_writer(*syms)
+    syms.flatten.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}                       # unless defined? @@hair_colors
+          @@#{sym} = nil                               #   @@hair_colors = nil
+        end                                            # end
+                                                       #
+        def self.#{sym}=(obj)                          # def self.hair_colors=(obj)
+          @@#{sym} = obj                               #   @@hair_colors = obj
+        end                                            # end
+                                                       #
+        def #{sym}=(obj)                               # def hair_colors=(obj)
+          @@#{sym} = obj                               #   @@hair_colors = obj
+        end                                            # end
+      EOS
+    end
+  end
+
+  def cattr_accessor(*syms)
+    cattr_reader(*syms)
+    cattr_writer(*syms)
+  end
+end

data/lib/extensions/mattr_accessor.rb

@@ -0,0 +1,55 @@
+# Extends the module object with module and instance accessors for class attributes, 
+# just like the native attr* accessors for instance attributes.
+#
+#  module AppConfiguration
+#    mattr_accessor :google_api_key
+#    self.google_api_key = "123456789"
+#
+#    mattr_accessor :paypal_url
+#    self.paypal_url = "www.sandbox.paypal.com"
+#  end
+#
+#  AppConfiguration.google_api_key = "overriding the api key!"
+class Module
+  def mattr_reader(*syms)
+    syms.each do |sym|
+      next if sym.is_a?(Hash)
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}  # unless defined? @@pagination_options
+          @@#{sym} = nil          #   @@pagination_options = nil
+        end                       # end
+                                  #
+        def self.#{sym}           # def self.pagination_options
+          @@#{sym}                #   @@pagination_options
+        end                       # end
+                                  #
+        def #{sym}                # def pagination_options
+          @@#{sym}                #   @@pagination_options
+        end                       # end
+      EOS
+    end
+  end
+  
+  def mattr_writer(*syms)
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__)
+        unless defined? @@#{sym}                       # unless defined? @@pagination_options
+          @@#{sym} = nil                               #   @@pagination_options = nil
+        end                                            # end
+                                                       #
+        def self.#{sym}=(obj)                          # def self.pagination_options=(obj)
+          @@#{sym} = obj                               #   @@pagination_options = obj
+        end                                            # end
+                                                       #
+        def #{sym}=(obj)                               # def pagination_options=(obj)
+          @@#{sym} = obj                               #   @@pagination_options = obj
+        end                                            # end
+      EOS
+    end
+  end
+  
+  def mattr_accessor(*syms)
+    mattr_reader(*syms)
+    mattr_writer(*syms)
+  end
+end

data/lib/workling.rb

@@ -0,0 +1,213 @@
+#
+#  I can haz am in your Workling are belong to us! 
+#
+def require_in_tree(name)
+  require File.join(File.dirname(__FILE__), name)
+end
+
+require_in_tree 'extensions/mattr_accessor' unless Module.respond_to?(:mattr_accessor)
+require_in_tree 'extensions/cattr_accessor' unless Class.respond_to?(:cattr_accessor)
+
+gem 'activesupport'
+require 'active_support/inflector'
+require 'active_support/core_ext/hash/keys'
+
+class Hash #:nodoc:
+  include ActiveSupport::CoreExtensions::Hash::Keys
+end
+
+require 'yaml'
+
+module Workling
+  class WorklingError < StandardError; end
+  class WorklingNotFoundError < WorklingError; end
+  class WorklingConnectionError < WorklingError; end
+  class QueueserverNotFoundError < WorklingError
+    def initialize
+      super "config/workling.yml configured to connect to queue server on #{ Workling.config[:listens_on] } for this environment. could not connect to queue server on this host:port. for starling users: pass starling the port with -p flag when starting it. If you don't want to use Starling, then explicitly set Workling::Remote.dispatcher (see README for an example)"
+    end
+  end
+  class ConfigurationError < WorklingError
+    def initialize
+      super File.exist?(Workling.path('config', 'starling.yml')) ? 
+        "config/starling.yml has been depracated. rename your config file to config/workling.yml then try again!" :
+        "config/workling.yml could not be loaded. check out README.markdown to see what this file should contain. "
+    end
+  end
+
+  def self.path(*args)
+    if defined?(RAILS_ROOT)
+      File.join(RAILS_ROOT, *args)
+    else
+      File.join(Dir.pwd, *args)
+    end
+  end
+
+  def self.env
+    @env ||= if defined?(RAILS_ENV)
+               RAILS_ENV.to_s
+             elsif defined?(RACK_ENV)
+               RACK_ENV.to_s
+             end
+  end
+
+  mattr_accessor :load_path
+  @@load_path = [ File.expand_path(path('app', 'workers')) ]
+
+  VERSION = "0.4.9" unless defined?(VERSION)
+
+  #
+  # determine the client to use if nothing is specifically set. workling will try to detect
+  # starling, spawn, or bj, in that order. if none of these are found, notremoterunner will
+  # be used. 
+  #
+  def self.select_default_client
+    if env == "test"
+      Workling::Clients::NotRemoteClient
+    elsif Workling::Clients::SpawnClient.installed?
+      Workling::Clients::SpawnClient
+    elsif Workling::Clients::BackgroundjobClient.installed?
+      Workling::Clients::BackgroundjobClient
+    else
+      Workling::Clients::NotRemoteClient
+    end
+  end
+
+  def self.clients
+    {
+      'amqp' => Workling::Clients::AmqpClient,
+      'amqp_exchange' => Workling::Clients::AmqpExchangeClient,
+      'memcache' => Workling::Clients::MemcacheQueueClient,
+      'starling' => Workling::Clients::MemcacheQueueClient,
+      'memory_queue' => Workling::Clients::MemoryQueueClient,
+      'sqs' => Workling::Clients::SqsClient,
+      'xmpp' => Workling::Clients::XmppClient,
+      'backgroundjob' => Workling::Clients::BackgroundjobClient,
+      'not_remote' => Workling::Clients::NotRemoteClient,
+      'not' => Workling::Clients::NotClient,
+      'spawn' => Workling::Clients::SpawnClient,
+      'thread' => Workling::Clients::ThreadClient,
+      'rudeq' => Workling::Clients::RudeQClient
+    }
+  end
+
+  def self.select_client
+    client_class = clients[Workling.config[:client]] || select_default_client
+    client_class.load
+    client_class
+  end
+
+  #
+  # this will build the client to use for job dispatching and retrieval
+  # The look up does the following:
+  #    1. if Workling::Remote.client is set explicitly then that client is used
+  #    2. if workling.yml (or whatever file is used) contains a client section then that is used
+  #    3. otherwise the default client is built using the Workling.select_and_build_default_client method
+  #
+  def self.select_and_build_client
+    select_client.new
+  end
+
+  #
+  # this will select the routing class
+  #
+  def self.select_and_build_routing
+    routing_class = {
+      'class_and_method' => Workling::Routing::ClassAndMethodRouting,
+      'static' => Workling::Routing::StaticRouting
+    }[Workling.config[:routing]] || Workling::Routing::ClassAndMethodRouting
+    routing_class.new
+  end
+
+  #
+  # this will build the invoker which will run the daemon
+  #
+  def self.select_and_build_invoker
+    invoker_class = {
+      'basic_poller' => Workling::Invokers::BasicPoller,
+      'thread_pool_poller' => Workling::Invokers::ThreadPoolPoller,
+      'threaded_poller' => Workling::Invokers::ThreadedPoller,
+
+      'eventmachine_subscriber' => Workling::Invokers::EventmachineSubscriber,
+      'looped_subscriber' => Workling::Invokers::LoopedSubscriber,
+      'amqp_single_subscriber' => Workling::Invokers::AmqpSingleSubscriber,
+    }[Workling.config[:invoker]] || Workling::Invokers::BasicPoller
+    invoker_class.new(select_and_build_routing, select_client)
+  end
+
+  #
+  # gets the worker instance, given a class. the optional method argument will cause an 
+  # exception to be raised if the worker instance does not respoind to said method. 
+  #
+  def self.find(clazz, method = nil)
+    begin
+      inst = clazz.to_s.camelize.constantize.new 
+    rescue NameError
+      raise_not_found(clazz, method)
+    end
+    raise_not_found(clazz, method) if method && !inst.respond_to?(method)
+    inst
+  end
+
+  # returns Workling::Return::Store.instance. 
+  def self.return
+    Workling::Return::Store.instance
+  end
+
+  #
+  #  returns a config hash. reads ./config/workling.yml
+  #
+  mattr_writer :config
+  def self.config
+    return @@config if defined?(@@config) && @@config
+
+    return nil unless File.exists?(config_path)
+
+    @@config ||= YAML.load_file(config_path)[env || 'development'].symbolize_keys
+    @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options]
+    @@config
+  end
+
+  mattr_writer :config_path
+  def self.config_path
+    return @@config_path if defined?(@@config_path) && @@config_path
+    @@config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
+  end
+
+  #
+  #  Raises exceptions thrown inside of the worker. normally, these are logged to 
+  #  logger.error. it's easy to miss these log calls while developing, though. 
+  #
+  mattr_writer :raise_exceptions
+  def raise_exceptions
+    return @@raise_exceptions if defined?(@@raise_exceptions)
+    @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
+  end
+
+  def self.raise_exceptions?
+    @@raise_exceptions
+  end
+
+  private
+    def self.raise_not_found(clazz, method)
+      raise Workling::WorklingNotFoundError.new("could not find #{ clazz }:#{ method } workling. ") 
+    end
+
+end
+
+require_in_tree "workling/discovery"
+require_in_tree "workling/base"
+require_in_tree "workling/remote"
+
+require_in_tree "workling/clients/base"
+require_in_tree "workling/clients/broker_base"
+require_in_tree "workling/invokers/base"
+require_in_tree "workling/return/store/base"
+require_in_tree "workling/routing/base"
+
+# load all possible extension classes
+["clients", "invokers", "return/store", "routing"].each do |e_dirs|
+  Dir.glob(File.join(File.dirname(__FILE__), "workling", e_dirs, "*.rb")).each do |rb_file|
+    require File.join(File.dirname(rb_file), File.basename(rb_file, ".rb"))
+  end
+end

data/lib/workling/base.rb

@@ -0,0 +1,110 @@
+#
+#  All worker classes must inherit from this class, and be saved in app/workers. 
+# 
+#  The Worker lifecycle: 
+#    The Worker is loaded once, at which point the instance method 'create' is called. 
+#
+#  Invoking Workers: 
+#    Calling async_my_method on the worker class will trigger background work.
+#    This means that the loaded Worker instance will receive a call to the method
+#    my_method(:uid => "thisjobsuid2348732947923"). 
+#
+#    The Worker method must have a single hash argument. Note that the job :uid will
+#    be merged into the hash. 
+#
+module Workling
+  class Base
+
+    cattr_writer :logger
+    def self.logger
+      @@logger ||= defined?(RAILS_DEFAULT_LOGGER) ? ::RAILS_DEFAULT_LOGGER : Logger.new($stdout)
+    end
+
+    def logger
+      self.class.logger
+    end
+
+    cattr_accessor :exposed_methods
+    @@exposed_methods ||= {}
+
+    def self.inherited(subclass)
+      Workling::Discovery.discovered << subclass
+    end
+
+    # expose a method using a custom queue name
+    def self.expose(method, options)
+      self.exposed_methods[method.to_s] = options[:as]
+    end
+
+    # identify the queue for a given method
+    def self.queue_for(method)
+      if self.exposed_methods.has_key?(method)
+        self.exposed_methods[method]
+      else
+        "#{ self.to_s.tableize }/#{ method }".split("/").join("__") # Don't split with : because it messes up memcache stats
+      end
+    end
+
+    # identify the method linked to the queue
+    def self.method_for(queue)
+      if self.exposed_methods.invert.has_key?(queue)
+        self.exposed_methods.invert[queue]
+      else
+        queue.split("__").last
+      end
+    end
+
+    def method_for(queue)
+      self.class.method_for(queue)
+    end
+
+
+    def initialize
+      super
+      
+      create
+    end
+
+    # Put worker initialization code in here. This is good for restarting jobs that
+    # were interrupted.
+    def create
+    end
+
+    # override this if you want to set up exception notification
+    def notify_exception(exception, method, options)
+      logger.error "WORKLING ERROR: runner could not invoke #{ self.class }:#{ method } with #{ options.inspect }. error was: #{ exception.inspect }\n #{ exception.backtrace.join("\n") }"
+    end
+
+    # takes care of suppressing remote errors but raising Workling::WorklingNotFoundError
+    # where appropriate. swallow workling exceptions so that everything behaves like remote code.
+    # otherwise StarlingRunner and SpawnRunner would behave too differently to NotRemoteRunner.
+    def dispatch_to_worker_method(method, options = {})
+      begin
+        options = default_options.merge(options)
+        self.send(method, options)
+      rescue Workling::WorklingError => e
+        raise e
+      rescue Exception => e
+        notify_exception e, method, options
+
+        # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
+        raise e if Workling.raise_exceptions?
+      end
+    end    
+
+    # supply default_options as a hash in classes that inherit Workling::Base
+    def default_options
+      {}
+    end
+  
+    # thanks to blaine cook for this suggestion.
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^asynch?_(.*)/
+        Workling::Remote.run(self.to_s.dasherize, $1, *args)
+      else
+        super
+      end
+    end
+
+  end
+end