derfred-workling 0.4.9.1 → 0.4.9.2

data/lib/workling/routing/class_and_method_routing.rb

@@ -0,0 +1,57 @@
+require 'workling/routing/base'
+
+#
+#  Holds a hash of routes. Each Worker method has a corresponding hash entry after building. 
+#
+module Workling
+  module Routing
+    class ClassAndMethodRouting < Base
+          
+      # initializes and builds routing hash. 
+      def initialize(*args)
+        super
+
+        build
+      end
+      
+      # returns the worker method name, given the routing string. 
+      def method_name(queue)
+        self[queue].method_for(queue)
+      end
+      
+      # returns the routing string, given a class and method. delegating. 
+      def queue_for(clazz, method)
+        ClassAndMethodRouting.queue_for(clazz, method)
+      end
+              
+      # returns the routing string, given a class and method.
+      def self.queue_for(clazz, method)
+        # this is lifted from the Rails constantize method
+        clazz = Object.module_eval("::#{clazz}") unless clazz.is_a?(Class)
+        clazz.queue_for method
+      end
+      
+      # returns all routed
+      def queue_names
+        self.keys
+      end
+      
+      # dare you to remove this! go on! 
+      def queue_names_routing_class(clazz)
+        self.select { |x, y|  y.is_a?(clazz) }.map { |x, y| x }
+      end
+
+      private
+        def build
+          Workling::Discovery.discovered.each do |clazz|
+            methods = clazz.public_instance_methods(false)
+            methods.each do |method|
+              next if method == 'create'  # Skip the create method
+              queue = queue_for(clazz, method)
+              self[queue] = clazz.new
+            end
+          end
+        end
+    end
+  end
+end

data/lib/workling/routing/static_routing.rb

@@ -0,0 +1,47 @@
+require 'workling/routing/base'
+
+#
+#  Holds a single route for a dedicated worker (if you want more worker processes, run more workers)
+#
+module Workling
+  module Routing
+    class StaticRouting < Base
+
+      #
+      # ./script/workling_client run -- <worker_class> <worker_method> <routing_key>      
+      # ./script/workling_client run -- <worker_class> <worker_method> <routing_key> <queue_name>
+      #
+      def initialize(*args)
+        @worker = args[0].constantize.new
+        @method_name = args[1]
+        @routing_key = args[2]
+
+        if(args.size==4)
+          @queue_name = args[3]
+        else
+          @queue_name = [@worker.class.to_s.tableize, @method_name, @routing_key].join("__")
+        end
+
+        # so routing[x] hash access works as expected
+        self.default = @worker
+
+        puts "** static routing: queue - #{@queue_name}, routing_key - #{@routing_key}, method - #{@method_name}, worker - #{@worker.class}"
+      end
+
+      # returns the worker method name, given the routing string. 
+      def method_name(queue=nil); @method_name; end
+
+      def routing_key_for; @routing_key; end
+
+      # returns the routing string, given a class and method. delegating.
+      # TODO - we can check for consistency here with clazz and methods vs. current configuration of this single route
+      def queue_for(clazz=nil, method=nil); @queue_name; end
+   
+      # returns array containing the single configured route
+      def queue_names; [@queue_name]; end
+      
+      # TODO - not sure what this is...
+      def queue_names_routing_class(clazz); @worker.class; end         
+    end
+  end
+end

data/lib/workling/rudeq/client.rb

@@ -0,0 +1,17 @@
+require 'workling/rudeq'
+
+module Workling
+  module Rudeq
+    class Client
+      attr_reader :queue
+      
+      def initialize 
+        @queue = Workling::Rudeq.config[:queue_class].constantize
+      end
+      
+      def method_missing(method, *args)
+        @queue.send(method, *args)
+      end
+    end
+  end
+end

data/lib/workling/rudeq/poller.rb

@@ -0,0 +1,116 @@
+require 'workling/rudeq'
+
+module Workling
+  module Rudeq
+    
+    class Poller
+      
+      cattr_accessor :sleep_time # Seconds to sleep before looping
+      cattr_accessor :reset_time # Seconds to wait while resetting connection
+
+      def initialize(routing)
+        Poller.sleep_time = Workling::Rudeq.config[:sleep_time] || 2
+        Poller.reset_time = Workling::Rudeq.config[:reset_time] || 30
+          
+        @routing = routing
+        @workers = ThreadGroup.new
+      end      
+      
+      def logger
+        Workling::Base.logger
+      end
+    
+      def listen
+                
+        # Allow concurrency for our tasks
+        ActiveRecord::Base.allow_concurrency = true
+
+        # Create a thread for each worker.
+        Workling::Discovery.discovered.each do |clazz|
+          logger.debug("Discovered listener #{clazz}")
+          @workers.add(Thread.new(clazz) { |c| clazz_listen(c) })
+        end
+        
+        # Wait for all workers to complete
+        @workers.list.each { |t| t.join }
+
+        # Clean up all the connections.
+        ActiveRecord::Base.verify_active_connections!
+      end
+      
+      # gracefully stop processing
+      def stop
+        @workers.list.each { |w| w[:shutdown] = true }
+      end
+      
+      ##
+      ## Thread procs
+      ##
+      
+      # Listen for one worker class
+      def clazz_listen(clazz)
+        
+        logger.debug("Listener thread #{clazz.name} started")
+           
+        # Read thread configuration if available
+        if Rudeq.config.has_key?(:listeners)
+          if Rudeq.config[:listeners].has_key?(clazz.to_s)
+            config = Rudeq.config[:listeners][clazz.to_s].symbolize_keys
+            thread_sleep_time = config[:sleep_time] if config.has_key?(:sleep_time)
+          end
+        end
+
+        hread_sleep_time ||= self.class.sleep_time
+        
+        connection = Workling::Rudeq::Client.new
+        puts "** Starting Workling::Rudeq::Client for #{clazz.name} queue"
+        
+        # Start dispatching those messages
+        while (!Thread.current[:shutdown]) do
+          begin
+            
+            # Keep MySQL connection alive
+            unless ActiveRecord::Base.connection.active?
+              unless ActiveRecord::Base.connection.reconnect!
+                logger.fatal("FAILED - Database not available")
+                break
+              end
+            end
+
+            # Dispatch and process the messages
+            n = dispatch!(connection, clazz)
+            logger.debug("Listener thread #{clazz.name} processed #{n.to_s} queue items") if n > 0
+            sleep(self.class.sleep_time) unless n > 0
+          end
+        end
+        
+        logger.debug("Listener thread #{clazz.name} ended")
+      end
+      
+      # Dispatcher for one worker class.
+      # Returns the number of worker methods called
+      def dispatch!(connection, clazz)
+        n = 0
+        for queue in @routing.queue_names_routing_class(clazz)
+          begin
+            result = connection.get(queue)
+            if result
+              n += 1
+              handler = @routing[queue]
+              method_name = @routing.method_name(queue)
+              logger.debug("Calling #{handler.class.to_s}\##{method_name}(#{result.inspect})")
+              handler.send(method_name, result)
+            end
+          rescue
+            logger.error("FAILED to connect with queue #{ queue }: #{ e } }")
+            raise e
+          rescue Object => e
+            logger.error("FAILED to process queue #{ queue }. #{ @routing[queue] } could not handle invocation of #{ @routing.method_name(queue) } with #{ result.inspect }: #{ e }.\n#{ e.backtrace.join("\n") }")
+          end
+        end
+        
+        return n
+      end
+    end
+  end
+end

data/lib/workling/rudeq.rb

@@ -0,0 +1,7 @@
+module Workling
+  module Rudeq
+    def self.config
+      @@config ||=  {:queue_class => "RudeQueue"}
+    end    
+  end
+end

data/lib/workling.rb

@@ -0,0 +1,195 @@
+#
+#  I can haz am in your Workling are belong to us! 
+#
+require 'mattr_accessor' unless Module.respond_to?(:mattr_accessor)
+require 'cattr_accessor' unless Class.respond_to?(:cattr_accessor)
+
+gem 'activesupport'
+require 'active_support/inflector'
+require 'active_support/core_ext/hash/keys'
+
+class Hash #:nodoc:
+  include ActiveSupport::CoreExtensions::Hash::Keys
+end
+
+require 'yaml'
+
+module Workling
+  class WorklingError < StandardError; end
+  class WorklingNotFoundError < WorklingError; end
+  class WorklingConnectionError < WorklingError; end
+  class QueueserverNotFoundError < WorklingError
+    def initialize
+      super "config/workling.yml configured to connect to queue server on #{ Workling.config[:listens_on] } for this environment. could not connect to queue server on this host:port. for starling users: pass starling the port with -p flag when starting it. If you don't want to use Starling, then explicitly set Workling::Remote.dispatcher (see README for an example)"
+    end
+  end
+
+  class ConfigurationError < WorklingError
+    def initialize
+      super File.exist?(Workling.path('config', 'starling.yml')) ? 
+        "config/starling.yml has been depracated. rename your config file to config/workling.yml then try again!" :
+        "config/workling.yml could not be loaded. check out README.markdown to see what this file should contain. "
+    end
+  end
+  
+  def self.path(*args)
+    if defined?(RAILS_ROOT)
+      File.join(RAILS_ROOT, *args)
+    else
+      File.join(Dir.pwd, *args)
+    end
+  end
+
+  def self.env
+    @env ||= if defined?(RAILS_ENV)
+               RAILS_ENV.to_s
+             elsif defined?(RACK_ENV)
+               RACK_ENV.to_s
+             end
+  end
+
+  mattr_accessor :load_path
+  @@load_path = [ File.expand_path(path('app', 'workers')) ]
+
+  VERSION = "0.4.9"
+
+  #
+  # determine the runner to use if nothing is specifically set. workling will try to detect
+  # starling, spawn, or bj, in that order. if none of these are found, notremoterunner will
+  # be used. 
+  #
+  # this can be overridden by setting Workling::Remote.dispatcher, eg:
+  #   Workling::Remote.dispatcher = Workling::Remote::Runners::StarlingRunner.new
+  #
+  def self.default_runner
+    if env == "test"
+      Workling::Remote::Runners::NotRemoteRunner.new
+    elsif starling_installed?
+      Workling::Remote::Runners::StarlingRunner.new
+    elsif spawn_installed?
+      Workling::Remote::Runners::SpawnRunner.new
+    elsif bj_installed?
+      Workling::Remote::Runners::BackgroundjobRunner.new
+    else
+      Workling::Remote::Runners::NotRemoteRunner.new
+    end
+  end
+  
+  #
+  # gets the worker instance, given a class. the optional method argument will cause an 
+  # exception to be raised if the worker instance does not respoind to said method. 
+  #
+  def self.find(clazz, method = nil)
+    begin
+      inst = clazz.to_s.camelize.constantize.new 
+    rescue NameError
+      raise_not_found(clazz, method)
+    end
+    raise_not_found(clazz, method) if method && !inst.respond_to?(method)
+    inst
+  end
+  
+  # returns Workling::Return::Store.instance. 
+  def self.return
+    Workling::Return::Store.instance
+  end
+
+  # is spawn installed?
+  def self.spawn_installed?
+    begin
+      require 'spawn'
+    rescue LoadError
+    end
+
+    Object.const_defined? "Spawn"
+  end
+
+  # is starling installed?  
+  def self.starling_installed?
+    begin
+      require 'starling' 
+    rescue LoadError
+    end
+      
+    Object.const_defined? "Starling"
+  end
+
+  # is bj installed?
+  def self.bj_installed?
+    Object.const_defined? "Bj"
+  end
+  
+  # Attempts to load the memcache-client gem
+  def self.try_load_a_memcache_client
+    begin
+      gem 'memcache-client'
+      require 'memcache'
+    rescue Gem::LoadError
+      Workling::Base.logger.info "WORKLING: couldn't find memcache-client. Install: \"gem install memcache-client\". "
+    end
+  end
+  
+  # attempts to load amqp and writes out descriptive error message if not present
+  def self.try_load_an_amqp_client
+    begin
+      require 'mq'
+    rescue Exception => e
+      raise WorklingError.new(
+        "WORKLING: couldn't find the ruby amqp client - you need it for the amqp runner. " \
+        "Install from github: gem sources -a http://gems.github.com/ && sudo gem install tmm1-amqp "
+      )
+    end
+  end
+
+  # attempts to load xmpp4r library and writes out descriptive error message if not present
+  def self.try_load_xmpp4r
+    begin
+      gem "xmpp4r"
+      require 'xmpp4r'
+      require "xmpp4r/pubsub"
+      require "xmpp4r/pubsub/helper/servicehelper.rb"
+      require "xmpp4r/pubsub/helper/nodebrowser.rb"
+      require "xmpp4r/pubsub/helper/nodehelper.rb"
+    rescue Exception => e
+      raise WorklingError.new("Couldnt load the XMPP library. check that you have the xmpp4r gem installed")
+    end
+  end
+
+
+  #
+  #  returns a config hash. reads ./config/workling.yml
+  #
+  def self.config
+    return @@config if defined?(@@config) && @@config
+
+    config_path = File.join(RAILS_ROOT, 'config', 'workling.yml')
+    return nil unless File.exists?(config_path)
+    
+    @@config ||= YAML.load_file(config_path)[RAILS_ENV || 'development'].symbolize_keys
+    @@config[:memcache_options].symbolize_keys! if @@config[:memcache_options]
+    @@config
+  end
+  
+  #
+  #  Raises exceptions thrown inside of the worker. normally, these are logged to 
+  #  logger.error. it's easy to miss these log calls while developing, though. 
+  #
+  mattr_accessor :raise_exceptions
+  @@raise_exceptions = (RAILS_ENV == "test" || RAILS_ENV == "development")
+  
+  def self.raise_exceptions?
+    @@raise_exceptions
+  end
+  
+  private
+    def self.raise_not_found(clazz, method)
+      raise Workling::WorklingNotFoundError.new("could not find #{ clazz }:#{ method } workling. ") 
+    end
+end
+
+require "workling/discovery"
+require "workling/base"
+require "workling/remote"
+require "workling/return/store/base"
+require "workling/return/store/memory_return_store"
+require "workling/return/store/starling_return_store"

data/lib/workling_server.rb

@@ -0,0 +1,108 @@
+require 'optparse'
+
+class WorklingServer
+  @@in_server_mode = false
+  def self.in_server_mode
+    @@in_server_mode
+  end
+
+  def self.in_server_mode=(server_mode)
+    @@in_server_mode = server_mode
+  end
+
+
+  def self.partition_options(args)
+    daemon = []
+    workling = []
+    split_point = args.index("--") || args.size
+    daemon = args[0...split_point] if split_point > 0
+    workling = args[(split_point+1)..-1] if split_point and split_point < args.size
+    [daemon, workling]
+  end
+
+  def self.partition_daemons_options(args)
+    standard_options = %W{start stop restart run zap -t --ontop -f --force -h --help --version}
+    pass_through = args.select { |a| standard_options.include? a }
+    custom_options = args.reject { |a| standard_options.include? a }
+
+    [pass_through, custom_options]
+  end
+
+  def self.parse_daemon_options(argv)
+    options = {}
+    pass_through, args = partition_daemons_options argv
+    opts = OptionParser.new do |opts|
+      opts.banner = 'Usage: myapp [options]'
+      opts.separator ''
+      opts.on('-a', '--app-name APP_NAME', String,"specify the process name") { |v| options[:app_name] = v }
+      opts.on('-d', '--dir DIR', String, "the directory to run in") { |v| options[:dir] = v }
+      opts.on('-m', '--monitor',"specify the process name") { |v| options[:monitor] = true }
+      opts.on('-t', '--ontop') { |k, v| pass_through << v  }
+    end
+    opts.parse!(partition_options(args).first)
+    options.merge(:ARGV => pass_through)
+  end
+
+  def self.parse_workling_options(args)
+    options = {}
+    opts = OptionParser.new do |opts|
+      opts.banner = 'Usage: myapp [options]'
+      opts.separator ''
+      opts.on('-c', '--client CLIENT', String,"specify the client class") { |v| options[:client_class] = v }
+      opts.on('-i', '--invoker INVOKER', String,"specify the invoker class") { |v| options[:invoker_class] = v }
+      opts.on('-r', '--routing ROUTING', String,"specify the routing class") { |v| options[:routing_class] = v }
+      opts.on('-l', '--load-path LOADPATH', String,"specify the load_path for the workers") { |v| options[:load_path] = v }
+      opts.on('-e', '--environment ENVIRONMENT', String,"specify the environment") { |v| options[:rails_env] = v }
+    end
+    opts.parse!(partition_options(args).last)
+    options
+  end
+
+
+  def self.build_poller(options)
+    require 'workling/remote'
+    ["remote/invokers/*.rb", "routing/*.rb"].each do |pattern|
+      Dir.glob(pattern).each do |f|
+        require File.join(File.dirname(f), File.basename(f, ".rb"))
+      end
+    end
+
+    routing_class = Object.module_eval("::#{options[:routing_class]}")
+    client_class = Object.module_eval("::#{options[:client_class]}")
+    invoker_class = Object.module_eval("::#{options[:invoker_class]}")
+    invoker_class.new(routing_class.new, client_class)
+  end
+
+
+  def self.run(options)
+    WorklingServer.in_server_mode = true
+
+    ENV["RAILS_ENV"] = options[:rails_env]
+    puts "=> Loading Rails with #{ENV["RAILS_ENV"]} environment..."
+
+    require options[:rails_root] + '/config/environment'
+
+    Workling.load_path = options[:load_path]
+
+    Workling::Discovery.discover!
+    Workling.config
+
+    poller = build_poller options
+
+    puts '** Rails loaded.'
+    puts "** Starting #{ poller.class }..."
+    puts '** Use CTRL-C to stop.'
+
+    ActiveRecord::Base.logger = Workling::Base.logger
+    ActionController::Base.logger = Workling::Base.logger
+
+    trap(:INT) { poller.stop; exit }
+
+    begin
+      poller.listen
+    ensure
+      puts '** No Worklings found.' if Workling::Discovery.discovered.blank?
+      puts '** Exiting'
+    end
+  end
+end

data/script/bj_invoker.rb

@@ -0,0 +1,11 @@
+@routing = Workling::Routing::ClassAndMethodRouting.new
+unnormalized = REXML::Text::unnormalize(STDIN.read)
+message, command, args = *unnormalized.match(/(^[^ ]*) (.*)/)
+options = Hash.from_xml(args)["hash"]
+
+if workling = @routing[command]
+  options = options.symbolize_keys
+  method_name = @routing.method_name(command)
+
+  workling.dispatch_to_worker_method(method_name, options)
+end

data/script/starling_status.rb

@@ -0,0 +1,37 @@
+require 'pp'
+
+puts '=> Loading Rails...'
+
+require File.dirname(__FILE__) + '/../config/environment'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/remote/invokers/basic_poller'
+require File.dirname(__FILE__) + '/../vendor/plugins/workling/lib/workling/routing/class_and_method_routing'
+
+puts '** Rails loaded.'
+
+trap(:INT) { exit }
+
+client = Workling::Clients::MemcacheQueueClient.new
+  
+begin
+  client.connect
+  client.reset
+  
+  client.stats # do this so that connection is shown as established below. 
+  
+  puts "Queue state:"
+  pp client.inspect
+  pp "Active?: #{client.active?}"
+  pp "Read Only?: #{client.readonly?}"
+  puts ""
+  puts "Servers:"
+  pp client.servers
+  puts ""
+  puts "Queue stats:"
+  pp client.stats
+
+  puts "\nThread Stats:"
+  pp Thread.list
+ensure
+  puts '** Exiting'
+  client.close
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: derfred-workling
 version: !ruby/object:Gem::Version 
-  version: 0.4.9.1
+  version: 0.4.9.2
 platform: ruby
 authors: 
 - Rany Keddo
@@ -26,6 +26,49 @@ files:
 - VERSION.yml
 - README.markdown
 - TODO.markdown
+- lib/cattr_accessor.rb
+- lib/mattr_accessor.rb
+- lib/rude_q/client.rb
+- lib/workling/base.rb
+- lib/workling/clients/amqp_client.rb
+- lib/workling/clients/amqp_exchange_client.rb
+- lib/workling/clients/base.rb
+- lib/workling/clients/memcache_queue_client.rb
+- lib/workling/clients/sqs_client.rb
+- lib/workling/clients/xmpp_client.rb
+- lib/workling/discovery.rb
+- lib/workling/remote/invokers/amqp_single_subscriber.rb
+- lib/workling/remote/invokers/base.rb
+- lib/workling/remote/invokers/basic_poller.rb
+- lib/workling/remote/invokers/eventmachine_subscriber.rb
+- lib/workling/remote/invokers/looped_subscriber.rb
+- lib/workling/remote/invokers/thread_pool_poller.rb
+- lib/workling/remote/invokers/threaded_poller.rb
+- lib/workling/remote/runners/amqp_exchange_runner.rb
+- lib/workling/remote/runners/backgroundjob_runner.rb
+- lib/workling/remote/runners/base.rb
+- lib/workling/remote/runners/client_runner.rb
+- lib/workling/remote/runners/not_remote_runner.rb
+- lib/workling/remote/runners/not_runner.rb
+- lib/workling/remote/runners/rudeq_runner.rb
+- lib/workling/remote/runners/spawn_runner.rb
+- lib/workling/remote/runners/starling_runner.rb
+- lib/workling/remote.rb
+- lib/workling/return/store/base.rb
+- lib/workling/return/store/iterator.rb
+- lib/workling/return/store/memory_return_store.rb
+- lib/workling/return/store/rudeq_return_store.rb
+- lib/workling/return/store/starling_return_store.rb
+- lib/workling/routing/base.rb
+- lib/workling/routing/class_and_method_routing.rb
+- lib/workling/routing/static_routing.rb
+- lib/workling/rudeq/client.rb
+- lib/workling/rudeq/poller.rb
+- lib/workling/rudeq.rb
+- lib/workling.rb
+- lib/workling_server.rb
+- script/bj_invoker.rb
+- script/starling_status.rb
 has_rdoc: true
 homepage: http://github.com/elecnix/workling
 post_install_message: