derfred-workling 0.4.9.2 → 0.4.9.3

data/bin/workling_client

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 require 'rubygems'
 require 'daemons'
-require 'workling_server'
+require File.join(File.dirname(__FILE__), '../lib/workling_daemon')
 
 daemon_options = {
   :app_name   => "workling",
@@ -11,18 +11,20 @@ daemon_options = {
   :multiple   => false,
   :backtrace  => true,
   :monitor    => false
-}.merge(WorklingServer.parse_daemon_options(ARGV))
+}.merge(WorklingDaemon.parse_daemon_options(ARGV))
 
 workling_options = {
   :client_class => "Workling::Clients::MemcacheQueueClient",
   :invoker_class => "Workling::Remote::Invokers::ThreadedPoller",
   :routing_class => "Workling::Routing::ClassAndMethodRouting",
   :rails_root => Dir.pwd,
-  :load_path => [ 'app/workers/**/*.rb' ],
-  :rails_env => (ENV['RAILS_ENV'] || "development").dup
-}.merge(WorklingServer.parse_workling_options(ARGV))
+  :load_path => ['app/workers/**/*.rb'],
+  :config_path => "config/workling.yml",
+  :rails_env => (ENV['RAILS_ENV'] || "development").dup,
+  :no_rails => false
+}.merge(WorklingDaemon.parse_workling_options(ARGV))
 
 Daemons.run_proc(daemon_options[:app_name], daemon_options) do
   Dir.chdir(workling_options[:rails_root])
-  WorklingServer.run(workling_options)
-end
+  WorklingDaemon.run(workling_options)
+end

data/lib/workling.rb

@@ -1,8 +1,12 @@
 #
 #  I can haz am in your Workling are belong to us! 
 #
-require 'mattr_accessor' unless Module.respond_to?(:mattr_accessor)
-require 'cattr_accessor' unless Class.respond_to?(:cattr_accessor)
+def require_in_tree(name)
+  require File.join(File.dirname(__FILE__), name)
+end
+
+require_in_tree 'extensions/mattr_accessor' unless Module.respond_to?(:mattr_accessor)
+require_in_tree 'extensions/cattr_accessor' unless Class.respond_to?(:cattr_accessor)
 
 gem 'activesupport'
 require 'active_support/inflector'
@@ -51,30 +55,72 @@ module Workling
   mattr_accessor :load_path
   @@load_path = [ File.expand_path(path('app', 'workers')) ]
 
-  VERSION = "0.4.9"
+  VERSION = "0.4.9" unless defined?(VERSION)
 
   #
-  # determine the runner to use if nothing is specifically set. workling will try to detect
+  # determine the client to use if nothing is specifically set. workling will try to detect
   # starling, spawn, or bj, in that order. if none of these are found, notremoterunner will
   # be used. 
   #
-  # this can be overridden by setting Workling::Remote.dispatcher, eg:
-  #   Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new
-  #
-  def self.default_runner
+  def self.select_and_build_default_client
     if env == "test"
-      Workling::Remote::Runners::NotRemoteRunner.new
-    elsif starling_installed?
-      Workling::Remote::Runners::StarlingRunner.new
-    elsif spawn_installed?
-      Workling::Remote::Runners::SpawnRunner.new
-    elsif bj_installed?
-      Workling::Remote::Runners::BackgroundjobRunner.new
+      Workling::Clients::NotRemoteClient.new
+    elsif Workling::Clients::SpawnClient.installed?
+      Workling::Clients::SpawnClient.new
+    elsif Workling::Clients::BackgroundjobClient.installed?
+      Workling::Clients::BackgroundjobClient.new
     else
-      Workling::Remote::Runners::NotRemoteRunner.new
+      Workling::Clients::NotRemoteClient.new
     end
   end
-  
+
+  #
+  # this will build the client to use for job dispatching and retrieval
+  # The look up does the following:
+  #    1. if Workling::Remote.client is set explicitly then that client is used
+  #    2. if workling.yml (or whatever file is used) contains a client section then that is used
+  #    3. otherwise the default client is built using the Workling.select_and_build_default_client method
+  #
+  def self.select_and_build_client
+    case(Workling.config[:client])
+    when 'amqp'
+      Workling::Clients::AmqpClient
+
+    when 'amqp_exchange'
+      Workling::Clients::AmqpExchangeClient
+
+    when 'memcache', 'starling'
+      Workling::Clients::MemcacheQueueClient
+
+    when 'memory_queue'   # this one is pretty useles...
+      Workling::Clients::MemoryQueueClient
+
+    when 'sqs'
+      Workling::Clients::SqsClient
+
+    when 'xmpp'
+      Workling::Clients::XmppClient
+
+    when 'backgroundjob'
+      Workling::Clients::BackgroundjobClient
+
+    when 'not_remote'
+      Workling::Clients::NotRemoteClient
+
+    when 'not'
+      Workling::Clients::NotClient
+
+    when 'spawn'
+      Workling::Clients::SpawnClient
+
+    when 'thread'
+      Workling::Clients::ThreadClient
+
+    else
+      select_and_build_default_client
+    end
+  end
+
   #
   # gets the worker instance, given a class. the optional method argument will cause an 
   # exception to be raised if the worker instance does not respoind to said method. 
@@ -88,108 +134,66 @@ module Workling
     raise_not_found(clazz, method) if method && !inst.respond_to?(method)
     inst
   end
-  
+
   # returns Workling::Return::Store.instance. 
   def self.return
     Workling::Return::Store.instance
   end
 
-  # is spawn installed?
-  def self.spawn_installed?
-    begin
-      require 'spawn'
-    rescue LoadError
-    end
-
-    Object.const_defined? "Spawn"
-  end
-
-  # is starling installed?  
-  def self.starling_installed?
-    begin
-      require 'starling' 
-    rescue LoadError
-    end
-      
-    Object.const_defined? "Starling"
-  end
-
-  # is bj installed?
-  def self.bj_installed?
-    Object.const_defined? "Bj"
-  end
-  
-  # Attempts to load the memcache-client gem
-  def self.try_load_a_memcache_client
-    begin
-      gem 'memcache-client'
-      require 'memcache'
-    rescue Gem::LoadError
-      Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
-    end
-  end
-  
-  # attempts to load amqp and writes out descriptive error message if not present
-  def self.try_load_an_amqp_client
-    begin
-      require 'mq'
-    rescue Exception => e
-      raise WorklingError.new(
-        "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
-        "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
-      )
-    end
-  end
-
-  # attempts to load xmpp4r library and writes out descriptive error message if not present
-  def self.try_load_xmpp4r
-    begin
-      gem "xmpp4r"
-      require 'xmpp4r'
-      require "xmpp4r/pubsub"
-      require "xmpp4r/pubsub/helper/servicehelper.rb"
-      require "xmpp4r/pubsub/helper/nodebrowser.rb"
-      require "xmpp4r/pubsub/helper/nodehelper.rb"
-    rescue Exception => e
-      raise WorklingError.new("Couldnt load the XMPP library. check that you have the xmpp4r gem installed")
-    end
-  end
-
-
   #
   #  returns a config hash. reads ./config/workling.yml
   #
+  mattr_writer :config
   def self.config
     return @@config if defined?(@@config) && @@config
 
-    config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
     return nil unless File.exists?(config_path)
-    
-    @@config ||= YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys
+
+    @@config ||= YAML.load_file(config_path)[env || 'development'].symbolize_keys
     @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options]
     @@config
   end
-  
+
+  mattr_writer :config_path
+  def self.config_path
+    return @@config_path if defined?(@@config_path) && @@config_path
+    @@config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
+  end
+
   #
   #  Raises exceptions thrown inside of the worker. normally, these are logged to 
   #  logger.error. it's easy to miss these log calls while developing, though. 
   #
-  mattr_accessor :raise_exceptions
-  @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
-  
+  mattr_writer :raise_exceptions
+  def raise_exceptions
+    return @@raise_exceptions if defined?(@@raise_exceptions)
+    @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
+  end
+
   def self.raise_exceptions?
     @@raise_exceptions
   end
-  
+
   private
     def self.raise_not_found(clazz, method)
       raise Workling::WorklingNotFoundError.new("could not find #{ clazz }:#{ method } workling. ") 
     end
+
 end
 
-require "workling/discovery"
-require "workling/base"
-require "workling/remote"
-require "workling/return/store/base"
-require "workling/return/store/memory_return_store"
-require "workling/return/store/starling_return_store"
+require_in_tree "workling/discovery"
+require_in_tree "workling/base"
+require_in_tree "workling/remote"
+
+require_in_tree "workling/clients/base"
+require_in_tree "workling/clients/broker_base"
+require_in_tree "workling/invokers/base"
+require_in_tree "workling/return/store/base"
+require_in_tree "workling/routing/base"
+
+# load all possible extension classes
+["clients", "invokers", "return/store", "routing"].each do |e_dirs|
+  Dir.glob(File.join(File.dirname(__FILE__), "workling", e_dirs, "*.rb")).each do |rb_file|
+    require File.join(File.dirname(rb_file), File.basename(rb_file, ".rb"))
+  end
+end

data/lib/workling/base.rb

@@ -1,5 +1,3 @@
-#require 'struct'
-
 #
 #  All worker classes must inherit from this class, and be saved in app/workers. 
 # 
@@ -16,9 +14,14 @@
 #
 module Workling
   class Base
+
     cattr_writer :logger
     def self.logger
-      @@logger ||= ::RAILS_DEFAULT_LOGGER
+      @@logger ||= defined?(RAILS_DEFAULT_LOGGER) ? ::RAILS_DEFAULT_LOGGER : Logger.new($stdout)
+    end
+
+    def logger
+      self.class.logger
     end
 
     cattr_accessor :exposed_methods
@@ -83,7 +86,6 @@ module Workling
         raise e
       rescue Exception => e
         notify_exception e, method, options
-        on_error(e) if respond_to?(:on_error)
 
         # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
         raise e if Workling.raise_exceptions?
@@ -103,13 +105,6 @@ module Workling
         super
       end
     end
-    
-    def self.logger
-      @logger ||= defined?(RAILS_DEFAULT_LOGGER) ? ::RAILS_DEFAULT_LOGGER : Logger.new($stdout)
-    end
 
-    def logger
-      self.class.logger
-    end
   end
 end

data/lib/workling/clients/amqp_client.rb

@@ -1,13 +1,21 @@
-require 'workling/clients/base'
-Workling.try_load_an_amqp_client
-
 #
 #  An Ampq client
 #
 module Workling
   module Clients
-    class AmqpClient < Workling::Clients::Base
-      
+    class AmqpClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          require 'mq'
+        rescue Exception => e
+          raise WorklingError.new(
+            "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+            "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+          )
+        end
+      end
+
       # starts the client. 
       def connect
         begin

data/lib/workling/clients/amqp_exchange_client.rb

@@ -1,13 +1,21 @@
-require 'workling/clients/base'
-Workling.try_load_an_amqp_client
-
 #
 #  An Ampq client
 #
 module Workling
   module Clients
-    class AmqpExchangeClient < Workling::Clients::Base
-      
+    class AmqpExchangeClient < Workling::Clients::BrokerBase
+
+      def self.load
+        begin
+          require 'mq'
+        rescue Exception => e
+          raise WorklingError.new(
+            "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+            "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+          )
+        end
+      end
+
       # starts the client. 
       def connect
         begin
@@ -16,22 +24,19 @@ module Workling
           raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.)")
         end
       end
-      
+
       # no need for explicit closing. when the event loop
       # terminates, the connection is closed anyway. 
       def close; true; end
-      
+
       # subscribe to a queue
       def subscribe(key)
         @amq.queue(key).subscribe do |header, body|
-
-          puts "***** received msg with header - #{header.inspect}"
-
           value = YAML.load(body)
           yield value
         end
       end
-      
+
       # request and retrieve work
       def retrieve(key)
           @amq.queue(key)
@@ -40,11 +45,14 @@ module Workling
       # publish message to exchange
       # using the specified routing key
       def request(exchange_name, value)
+        exchange_name = "amq.topic"
+
         key = value.delete(:routing_key)
         msg = YAML.dump(value)
         exchange = @amq.topic(exchange_name)
         exchange.publish(msg, :routing_key => key)
       end
+
     end
   end
 end

data/lib/workling/clients/backgroundjob_client.rb

@@ -0,0 +1,25 @@
+module Workling
+  module Clients
+    class BackgroundjobClient < Workling::Clients::Base
+
+      def self.installed?
+        Object.const_defined? "Bj"
+      end
+
+      #  passes the job to bj by serializing the options to xml and passing them to
+      #  ./script/bj_invoker.rb, which in turn routes the deserialized args to the
+      #  appropriate worker. 
+      def dispatch(clazz, method, options = {})
+        stdin = Workling::Remote.routing.queue_for(clazz, method) + 
+                " " + 
+                options.to_xml(:indent => 0, :skip_instruct => true)
+                
+        Bj.submit "./script/runner ./script/bj_invoker.rb", 
+          :stdin => stdin
+        
+        return nil # that means nothing!
+      end
+
+    end
+  end
+end