derfred-workling 0.4.9.2 → 0.4.9.3

data/lib/workling/remote.rb

@@ -1,9 +1,3 @@
-require "workling/remote/runners/not_remote_runner"
-require "workling/remote/runners/spawn_runner"
-require "workling/remote/runners/starling_runner"
-require "workling/remote/runners/backgroundjob_runner"
-require "workling/remote/invokers/threaded_poller"
-
 require 'digest/md5'
 
 #
@@ -12,43 +6,18 @@ require 'digest/md5'
 module Workling
   module Remote
 
-    # Select which invoke to load based on what is defined in workling.yml. Defaults to thread_poller
-    # if none are specified.
-    def self.select_invoker
-      case(Workling.config[:invoker])
-      when 'basic_poller'
-        Workling::Remote::Invokers::BasicPoller
-
-      when 'thread_pool_poller'
-        Workling::Remote::Invokers::ThreadPoolPoller
-
-      when 'eventmachine_subscriber'
-        Workling::Remote::Invokers::EventmachineSubscriber
-
-      when 'threaded_poller', nil
-        Workling::Remote::Invokers::ThreadedPoller
-
-      else
-        Workling.logger.error("Nothing is known about #{Workling.config[:invoker]} defaulting to thread_poller")
-        Workling::Remote::Invokers::ThreadedPoller
-      end
+    # which object to use for routing
+    mattr_writer :routing
+    def self.routing
+      @@routing ||= Workling::Routing::ClassAndMethodRouting.new
     end
 
-    # set the desired runner here. this is initialized with Workling.default_runner. 
-    mattr_accessor :dispatcher
-
-    # set the desired invoker. this class grabs work from the job broker and executes it.
-    mattr_accessor :invoker
-    @@invoker ||= self.select_invoker
-
-    mattr_accessor :routing
-    @@routing ||= Workling::Routing::ClassAndMethodRouting
-
-    # retrieve the dispatcher or instantiate it using the defaults
-    def self.dispatcher
-      @@dispatcher ||= Workling.default_runner
+    # which client to use for dispatching
+    mattr_accessor :client
+    def self.client
+      @@client ||= Workling.select_and_build_client
     end
-    
+
     # generates a unique identifier for this particular job. 
     def self.generate_uid(clazz, method)
       uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
@@ -61,7 +30,7 @@ module Workling
       uid = Workling::Remote.generate_uid(clazz, method)
       options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
       Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist. 
-      dispatcher.run(clazz, method, options)
+      client.dispatch(clazz, method, options)
       uid
     end
 

data/lib/workling/return/store/base.rb

@@ -6,36 +6,36 @@ module Workling
   module Return
     module Store
       mattr_accessor :instance
-      
+
       # set a value in the store with the given key. delegates to the returnstore. 
       def self.set(key, value)
         self.instance.set(key, value)
       end
-      
+
       # get a value from the store. this should be destructive. delegates to the returnstore. 
       def self.get(key)
         self.instance.get(key)
       end
-      
+
       #
       #  Base Class for Return Stores. Subclasses need to implement set and get. 
       #
       class Base
-        
+
         # set a value in the store with the given key. 
         def set(key, value)
           raise NotImplementedError.new("set(key, value) not implemented in #{ self.class }")
         end
-      
+
         # get a value from the store. this should be destructive.
         def get(key)
           raise NotImplementedError.new("get(key) not implemented in #{ self.class }")
         end
-        
+
         def iterator(key)
           Workling::Return::Store::Iterator.new(key)
         end
-        
+
       end
     end
   end

data/lib/workling/return/store/memory_return_store.rb

@@ -1,5 +1,3 @@
-require 'workling/return/store/base'
-
 #
 #  Stores directly into memory. This is for tests only - not for production use. aight?
 #

data/lib/workling/return/store/starling_return_store.rb

@@ -1,6 +1,3 @@
-require 'workling/return/store/base'
-require 'workling/clients/memcache_queue_client'
-
 #
 #  Recommended Return Store if you are using the Starling Runner. This
 #  Simply sets and gets values against queues. 'key' is the name of the respective Queue. 
@@ -9,22 +6,24 @@ module Workling
   module Return
     module Store
       class StarlingReturnStore < Base
+
         cattr_accessor :client
-        
+
         def initialize
           self.client = Workling::Clients::MemcacheQueueClient.new
           self.client.connect
         end
-        
+
         # set a value in the queue 'key'. 
         def set(key, value)
-          self.class.client.set(key, value)
+          self.client.set(key, value)
         end
-        
+
         # get a value from starling queue 'key'.
         def get(key)
-          self.class.client.get(key)
+          self.client.get(key)
         end
+
       end
     end
   end

data/lib/workling/routing/base.rb

@@ -5,12 +5,9 @@
 module Workling
   module Routing
     class Base < Hash
-#      @@logger ||= ::RAILS_DEFAULT_LOGGER   
-#      cattr_accessor :logger
-      
       def method_name
         raise Exception.new("method_name not implemented.")
       end
     end
   end
-end
+end

data/lib/workling/routing/class_and_method_routing.rb

@@ -1,5 +1,3 @@
-require 'workling/routing/base'
-
 #
 #  Holds a hash of routes. Each Worker method has a corresponding hash entry after building. 
 #

data/lib/workling/routing/static_routing.rb

@@ -1,5 +1,3 @@
-require 'workling/routing/base'
-
 #
 #  Holds a single route for a dedicated worker (if you want more worker processes, run more workers)
 #
@@ -24,8 +22,6 @@ module Workling
 
         # so routing[x] hash access works as expected
         self.default = @worker
-
-        puts "** static routing: queue - #{@queue_name}, routing_key - #{@routing_key}, method - #{@method_name}, worker - #{@worker.class}"
       end
 
       # returns the worker method name, given the routing string. 

data/lib/{workling_server.rb → workling_daemon.rb}

@@ -1,15 +1,6 @@
 require 'optparse'
 
-class WorklingServer
-  @@in_server_mode = false
-  def self.in_server_mode
-    @@in_server_mode
-  end
-
-  def self.in_server_mode=(server_mode)
-    @@in_server_mode = server_mode
-  end
-
+class WorklingDaemon
 
   def self.partition_options(args)
     daemon = []
@@ -48,11 +39,13 @@ class WorklingServer
     opts = OptionParser.new do |opts|
       opts.banner = 'Usage: myapp [options]'
       opts.separator ''
-      opts.on('-c', '--client CLIENT', String,"specify the client class") { |v| options[:client_class] = v }
-      opts.on('-i', '--invoker INVOKER', String,"specify the invoker class") { |v| options[:invoker_class] = v }
-      opts.on('-r', '--routing ROUTING', String,"specify the routing class") { |v| options[:routing_class] = v }
-      opts.on('-l', '--load-path LOADPATH', String,"specify the load_path for the workers") { |v| options[:load_path] = v }
-      opts.on('-e', '--environment ENVIRONMENT', String,"specify the environment") { |v| options[:rails_env] = v }
+      opts.on('-n', '--no_rails', "do not load Rails") { |v| options[:no_rails] = true }
+      opts.on('-c', '--client CLIENT', String, "specify the client class") { |v| options[:client_class] = v }
+      opts.on('-i', '--invoker INVOKER', String, "specify the invoker class") { |v| options[:invoker_class] = v }
+      opts.on('-r', '--routing ROUTING', String, "specify the routing class") { |v| options[:routing_class] = v }
+      opts.on('-l', '--load-path LOADPATH', String, "specify the load_path for the workers") { |v| options[:load_path] = v }
+      opts.on('-f', '--config-path CONFIGPATH', String, "specify the path to the workling.yml file") { |v| options[:config_path] = v }
+      opts.on('-e', '--environment ENVIRONMENT', String, "specify the environment") { |v| options[:rails_env] = v }
     end
     opts.parse!(partition_options(args).last)
     options
@@ -60,7 +53,7 @@ class WorklingServer
 
 
   def self.build_poller(options)
-    require 'workling/remote'
+    require File.join(File.dirname(__FILE__), 'workling/remote')
     ["remote/invokers/*.rb", "routing/*.rb"].each do |pattern|
       Dir.glob(pattern).each do |f|
         require File.join(File.dirname(f), File.basename(f, ".rb"))
@@ -70,19 +63,25 @@ class WorklingServer
     routing_class = Object.module_eval("::#{options[:routing_class]}")
     client_class = Object.module_eval("::#{options[:client_class]}")
     invoker_class = Object.module_eval("::#{options[:invoker_class]}")
+
+    client_class.load
+
     invoker_class.new(routing_class.new, client_class)
   end
 
 
   def self.run(options)
-    WorklingServer.in_server_mode = true
-
-    ENV["RAILS_ENV"] = options[:rails_env]
-    puts "=> Loading Rails with #{ENV["RAILS_ENV"]} environment..."
-
-    require options[:rails_root] + '/config/environment'
+    if options[:no_rails]
+      # if rails is not booted we need to pull in the workling requires manually
+      require File.join(File.dirname(__FILE__), "workling")
+    else
+      ENV["RAILS_ENV"] = options[:rails_env]
+      puts "=> Loading Rails with #{ENV["RAILS_ENV"]} environment..."
+      require options[:rails_root] + '/config/environment'
+    end
 
     Workling.load_path = options[:load_path]
+    Workling.config_path = options[:config_path]
 
     Workling::Discovery.discover!
     Workling.config
@@ -93,16 +92,19 @@ class WorklingServer
     puts "** Starting #{ poller.class }..."
     puts '** Use CTRL-C to stop.'
 
-    ActiveRecord::Base.logger = Workling::Base.logger
-    ActionController::Base.logger = Workling::Base.logger
+    unless options[:no_rails]
+      ActiveRecord::Base.logger = Workling::Base.logger
+      ActionController::Base.logger = Workling::Base.logger
+    end
 
     trap(:INT) { poller.stop; exit }
 
     begin
       poller.listen
     ensure
-      puts '** No Worklings found.' if Workling::Discovery.discovered.blank?
+      puts '** No Worklings found.' if Workling::Discovery.discovered.empty?
       puts '** Exiting'
     end
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: derfred-workling
 version: !ruby/object:Gem::Version 
-  version: 0.4.9.2
+  version: 0.4.9.3
 platform: ruby
 authors: 
 - Rany Keddo
@@ -26,51 +26,44 @@ files:
 - VERSION.yml
 - README.markdown
 - TODO.markdown
-- lib/cattr_accessor.rb
-- lib/mattr_accessor.rb
-- lib/rude_q/client.rb
+- lib/extensions/cattr_accessor.rb
+- lib/extensions/mattr_accessor.rb
 - lib/workling/base.rb
 - lib/workling/clients/amqp_client.rb
 - lib/workling/clients/amqp_exchange_client.rb
+- lib/workling/clients/backgroundjob_client.rb
 - lib/workling/clients/base.rb
+- lib/workling/clients/broker_base.rb
 - lib/workling/clients/memcache_queue_client.rb
+- lib/workling/clients/memory_queue_client.rb
+- lib/workling/clients/not_client.rb
+- lib/workling/clients/not_remote_client.rb
+- lib/workling/clients/spawn_client.rb
 - lib/workling/clients/sqs_client.rb
+- lib/workling/clients/thread_client.rb
 - lib/workling/clients/xmpp_client.rb
 - lib/workling/discovery.rb
-- lib/workling/remote/invokers/amqp_single_subscriber.rb
-- lib/workling/remote/invokers/base.rb
-- lib/workling/remote/invokers/basic_poller.rb
-- lib/workling/remote/invokers/eventmachine_subscriber.rb
-- lib/workling/remote/invokers/looped_subscriber.rb
-- lib/workling/remote/invokers/thread_pool_poller.rb
-- lib/workling/remote/invokers/threaded_poller.rb
-- lib/workling/remote/runners/amqp_exchange_runner.rb
-- lib/workling/remote/runners/backgroundjob_runner.rb
-- lib/workling/remote/runners/base.rb
-- lib/workling/remote/runners/client_runner.rb
-- lib/workling/remote/runners/not_remote_runner.rb
-- lib/workling/remote/runners/not_runner.rb
-- lib/workling/remote/runners/rudeq_runner.rb
-- lib/workling/remote/runners/spawn_runner.rb
-- lib/workling/remote/runners/starling_runner.rb
+- lib/workling/invokers/amqp_single_subscriber.rb
+- lib/workling/invokers/base.rb
+- lib/workling/invokers/basic_poller.rb
+- lib/workling/invokers/eventmachine_subscriber.rb
+- lib/workling/invokers/looped_subscriber.rb
+- lib/workling/invokers/thread_pool_poller.rb
+- lib/workling/invokers/threaded_poller.rb
 - lib/workling/remote.rb
 - lib/workling/return/store/base.rb
 - lib/workling/return/store/iterator.rb
 - lib/workling/return/store/memory_return_store.rb
-- lib/workling/return/store/rudeq_return_store.rb
 - lib/workling/return/store/starling_return_store.rb
 - lib/workling/routing/base.rb
 - lib/workling/routing/class_and_method_routing.rb
 - lib/workling/routing/static_routing.rb
-- lib/workling/rudeq/client.rb
-- lib/workling/rudeq/poller.rb
-- lib/workling/rudeq.rb
 - lib/workling.rb
-- lib/workling_server.rb
-- script/bj_invoker.rb
-- script/starling_status.rb
+- lib/workling_daemon.rb
+- contrib/bj_invoker.rb
+- contrib/starling_status.rb
 has_rdoc: true
-homepage: http://github.com/elecnix/workling
+homepage: http://github.com/derfred/workling
 post_install_message: 
 rdoc_options: 
 - --inline-source

data/lib/rude_q/client.rb

@@ -1,11 +0,0 @@
-#
-#  A RudeQ client that behvaes somewhat like memcache-client
-#
-module RudeQ
-  class Client    
-    def initialize(*args); super(); end   
-    def set(key, value); RudeQueue.set(key, value); end;
-    def get(key); RudeQueue.get(key); end;
-    def stats; ActiveRecord::Base.connection; end
-  end
-end

data/lib/workling/remote/invokers/amqp_single_subscriber.rb

@@ -1,45 +0,0 @@
-require 'eventmachine'
-require 'workling/remote/invokers/base'
-
-#
-# TODO - Subscribes a single worker to a single queue 
-# 
-module Workling
-  module Remote
-    module Invokers
-      class AmqpSingleSubscriber < Workling::Remote::Invokers::Base
-        
-        def initialize(routing, client_class)
-          super
-        end
-        
-        #
-        # Starts EM loop and sets up subscription callback for the worker
-        # Create the queue and bind to exchange using the routing key
-        #
-        def listen
-          EM.run do
-            connect do
-              queue_name = @routing.queue_for
-              routing_key = @routing.routing_key_for
-
-              # temp stuff to hook the queues and exchanges up
-              # wildcard routing - # (match all)
-              exch = MQ.topic
-              q = MQ.queue(queue_name)
-              q.bind(exch, :key => routing_key)
-
-              @client.subscribe(queue_name) do |args|
-                run(queue_name, args)
-              end
-            end
-          end
-        end
-                
-        def stop
-          EM.stop if EM.reactor_running?
-        end
-      end
-    end
-  end
-end

data/lib/workling/remote/invokers/base.rb

@@ -1,124 +0,0 @@
-#
-#  Invokers are responsible for 
-#
-#      1. grabbing work off a job broker (such as a starling or rabbitmq server).
-#      2. routing (mapping) that work onto the correct worker method. 
-#      3. invoking the worker method, passing any arguments that came off the broker.
-#
-#   Invokers should implement their own concurrency strategies. For example, 
-#   The there is a ThreadedPoller which starts a thread for each Worker class.
-#
-#   This base Invoker class defines the methods an Invoker needs to implement. 
-#  
-module Workling
-  module Remote
-    module Invokers
-      class Base
-        
-        attr_accessor :sleep_time, :reset_time
-        
-        #
-        #  call up with super in the subclass constructor.
-        #
-        def initialize(routing, client_class)
-          @routing = routing
-          @client_class = client_class
-          @sleep_time = Workling.config[:sleep_time] || 2
-          @reset_time = Workling.config[:reset_time] || 30
-          @@mutex ||= Mutex.new
-        end
-        
-        #
-        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
-        #
-        def listen
-          raise NotImplementedError.new("Implement listen() in your Invoker. ")
-        end        
-        
-        #
-        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
-        #  to finish. 
-        #
-        def stop
-          raise NotImplementedError.new("Implement stop() in your Invoker. ")
-        end
-        
-        # 
-        #  Runs the worker method, given
-        #
-        #      type: the worker route
-        #      args: the arguments to be passed into the worker method.
-        #
-        def run(type, args)
-          worker = @routing[type]
-          method = @routing.method_name(type)
-          worker.dispatch_to_worker_method(method, args)
-        end
-        
-        # returns the Workling::Base.logger
-        def logger; Workling::Base.logger; end
-        
-        protected
-        
-          # handle opening and closing of client. pass code block to this method. 
-          def connect
-            @client = @client_class.new
-            @client.connect
-            
-            begin
-              yield
-            ensure
-              @client.close
-              ActiveRecord::Base.verify_active_connections!
-            end
-          end
-
-          #
-          #  Loops through the available routes, yielding for each route. 
-          #  This continues until @shutdown is set on this instance. 
-          #
-          def loop_routes
-            while(!@shutdown) do
-              ensure_activerecord_connection
-              
-              routes.each do |route|
-                break if @shutdown
-                yield route
-              end
-              
-              sleep self.sleep_time
-            end
-          end
-
-          #
-          #  Returns the complete set of active routes
-          #
-          def routes
-            @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
-          end
-          
-          # Thanks for this Brent! 
-          #
-          #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
-          #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
-          #     to wrap the code that checks for a connection in in a mutex.
-          #
-          #     ....I noticed this while working with a multi-core machine that 
-          #     was spawning multiple workling threads. Some of my workling 
-          #     threads would hit serious issues at this block of code without 
-          #     the mutex.            
-          #
-          def ensure_activerecord_connection
-            @@mutex.synchronize do 
-              unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
-                unless ActiveRecord::Base.connection.reconnect!
-                  logger.fatal("Failed - Database not available!")
-                  break
-                end
-              end
-            end            
-          end
-      end
-    end
-  end
-end