derfred-workling 0.4.9.1 → 0.4.9.2

data/lib/workling/clients/xmpp_client.rb

@@ -0,0 +1,100 @@
+require 'workling/clients/base'
+Workling.try_load_xmpp4r
+
+#
+#  An XMPP client
+#
+#  How to use: this client requires the xmpp4r gem
+#
+#  in the config/environments/development.rb file (or production.rb etc)
+#
+#    Workling::Remote::Runners::ClientRunner.client = Workling::Clients::XmppClient.new
+#    Workling::Remote.dispatcher = Workling::Remote::Runners::ClientRunner.new        # dont use the standard runner
+#    Workling::Remote.invoker = Workling::Remote::Invokers::LoopedSubscriber          # does not work with the EventmachineSubscriber Invoker
+#
+#  furthermore in the workling.yml file you need to set up the server details for your XMPP server
+#
+#    development:
+#      listens_on: "localhost:22122"
+#      jabber_id: "sub@localhost/laptop"
+#      jabber_server: "localhost"
+#      jabber_password: "sub"
+#      jabber_service: "pubsub.derfredtop.local"
+#
+#  for details on how to configure your XMPP server (ejabberd) check out the following howto:
+#
+#    http://keoko.wordpress.com/2008/12/17/xmpp-pubsub-with-ejabberd-and-xmpp4r/
+#
+#
+#  finally you need to expose your worker methods to XMPP nodes like so:
+# 
+#    class NotificationWorker < Workling::Base
+# 
+#      expose :receive_notification, :as => "/home/localhost/pub/sub"
+# 
+#      def receive_notification(input)
+#        # something here
+#      end
+# 
+#    end
+#
+
+
+
+module Workling
+  module Clients
+    class XmppClient < Workling::Clients::Base
+
+      # starts the client. 
+      def connect
+        begin
+          @client = Jabber::Client.new Workling.config[:jabber_id]
+          @client.connect Workling.config[:jabber_server]
+          @client.auth Workling.config[:jabber_password]
+          @client.send Jabber::Presence.new.set_type(:available)
+          @pubsub = Jabber::PubSub::ServiceHelper.new(@client, Workling.config[:jabber_service])
+          unsubscribe_from_all                # make sure there are no open subscriptions, could cause multiple delivery of notifications, as they are persistent
+        rescue
+          raise WorklingError.new("couldn't connect to the jabber server")
+        end
+      end
+
+      # disconnect from the server
+      def close
+        @client.close
+      end
+
+      # subscribe to a queue
+      def subscribe(key)
+        @pubsub.subscribe_to(key)
+
+        # filter out the subscription notification message that was generated by subscribing to the node
+        @pubsub.get_subscriptions_from_all_nodes()
+
+        @pubsub.add_event_callback do |event|
+          event.payload.each do |e|
+            e.children.each do |child|
+              yield Hash.from_xml(child.children.first.to_s) if child.name == 'item'
+            end
+          end
+        end
+      end
+
+      # request and retrieve work
+      def retrieve(key)
+        @pubsub.get_items_from(key, 1)
+      end
+
+      def request(key, value)
+        @pubsub.publish_item_to(key, value)
+      end
+
+      private
+        def unsubscribe_from_all
+          @pubsub.get_subscriptions_from_all_nodes.each do |subscription|
+            @pubsub.unsubscribe_from subscription.node
+          end
+        end
+    end
+  end
+end

data/lib/workling/discovery.rb

@@ -0,0 +1,16 @@
+#
+#  Discovery is responsible for loading workers in app/workers.
+#
+module Workling
+  class Discovery
+    cattr_accessor :discovered
+    @@discovered = []
+
+    # requires worklings so that they are added to routing.
+    def self.discover!
+      Workling.load_path.each do |p|
+        Dir.glob(p).each { |wling| require wling }
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/amqp_single_subscriber.rb

@@ -0,0 +1,45 @@
+require 'eventmachine'
+require 'workling/remote/invokers/base'
+
+#
+# TODO - Subscribes a single worker to a single queue 
+# 
+module Workling
+  module Remote
+    module Invokers
+      class AmqpSingleSubscriber < Workling::Remote::Invokers::Base
+        
+        def initialize(routing, client_class)
+          super
+        end
+        
+        #
+        # Starts EM loop and sets up subscription callback for the worker
+        # Create the queue and bind to exchange using the routing key
+        #
+        def listen
+          EM.run do
+            connect do
+              queue_name = @routing.queue_for
+              routing_key = @routing.routing_key_for
+
+              # temp stuff to hook the queues and exchanges up
+              # wildcard routing - # (match all)
+              exch = MQ.topic
+              q = MQ.queue(queue_name)
+              q.bind(exch, :key => routing_key)
+
+              @client.subscribe(queue_name) do |args|
+                run(queue_name, args)
+              end
+            end
+          end
+        end
+                
+        def stop
+          EM.stop if EM.reactor_running?
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/base.rb

@@ -0,0 +1,124 @@
+#
+#  Invokers are responsible for 
+#
+#      1. grabbing work off a job broker (such as a starling or rabbitmq server).
+#      2. routing (mapping) that work onto the correct worker method. 
+#      3. invoking the worker method, passing any arguments that came off the broker.
+#
+#   Invokers should implement their own concurrency strategies. For example, 
+#   The there is a ThreadedPoller which starts a thread for each Worker class.
+#
+#   This base Invoker class defines the methods an Invoker needs to implement. 
+#  
+module Workling
+  module Remote
+    module Invokers
+      class Base
+        
+        attr_accessor :sleep_time, :reset_time
+        
+        #
+        #  call up with super in the subclass constructor.
+        #
+        def initialize(routing, client_class)
+          @routing = routing
+          @client_class = client_class
+          @sleep_time = Workling.config[:sleep_time] || 2
+          @reset_time = Workling.config[:reset_time] || 30
+          @@mutex ||= Mutex.new
+        end
+        
+        #
+        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
+        #
+        def listen
+          raise NotImplementedError.new("Implement listen() in your Invoker. ")
+        end        
+        
+        #
+        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
+        #  to finish. 
+        #
+        def stop
+          raise NotImplementedError.new("Implement stop() in your Invoker. ")
+        end
+        
+        # 
+        #  Runs the worker method, given
+        #
+        #      type: the worker route
+        #      args: the arguments to be passed into the worker method.
+        #
+        def run(type, args)
+          worker = @routing[type]
+          method = @routing.method_name(type)
+          worker.dispatch_to_worker_method(method, args)
+        end
+        
+        # returns the Workling::Base.logger
+        def logger; Workling::Base.logger; end
+        
+        protected
+        
+          # handle opening and closing of client. pass code block to this method. 
+          def connect
+            @client = @client_class.new
+            @client.connect
+            
+            begin
+              yield
+            ensure
+              @client.close
+              ActiveRecord::Base.verify_active_connections!
+            end
+          end
+
+          #
+          #  Loops through the available routes, yielding for each route. 
+          #  This continues until @shutdown is set on this instance. 
+          #
+          def loop_routes
+            while(!@shutdown) do
+              ensure_activerecord_connection
+              
+              routes.each do |route|
+                break if @shutdown
+                yield route
+              end
+              
+              sleep self.sleep_time
+            end
+          end
+
+          #
+          #  Returns the complete set of active routes
+          #
+          def routes
+            @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
+          end
+          
+          # Thanks for this Brent! 
+          #
+          #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
+          #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
+          #     to wrap the code that checks for a connection in in a mutex.
+          #
+          #     ....I noticed this while working with a multi-core machine that 
+          #     was spawning multiple workling threads. Some of my workling 
+          #     threads would hit serious issues at this block of code without 
+          #     the mutex.            
+          #
+          def ensure_activerecord_connection
+            @@mutex.synchronize do 
+              unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
+                unless ActiveRecord::Base.connection.reconnect!
+                  logger.fatal("Failed - Database not available!")
+                  break
+                end
+              end
+            end            
+          end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/basic_poller.rb

@@ -0,0 +1,41 @@
+require 'workling/remote/invokers/base'
+
+#
+#  A basic polling invoker. 
+#  
+module Workling
+  module Remote
+    module Invokers
+      class BasicPoller < Workling::Remote::Invokers::Base
+        
+        #
+        #  set up client, sleep time
+        #
+        def initialize(routing, client_class)
+          super
+        end
+        
+        #
+        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
+        #
+        def listen
+          connect do
+            loop_routes do |route|
+              if args = @client.retrieve(route)
+                run(route, args)
+              end
+            end
+          end
+        end
+        
+        #
+        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
+        #  to finish. 
+        #
+        def stop
+          @shutdown = true
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/eventmachine_subscriber.rb

@@ -0,0 +1,41 @@
+require 'eventmachine'
+require 'workling/remote/invokers/base'
+
+#
+#  Subscribes the workers to the correct queues. 
+# 
+module Workling
+  module Remote
+    module Invokers
+      class EventmachineSubscriber < Workling::Remote::Invokers::Base
+        
+        def initialize(routing, client_class)
+          super
+        end
+        
+        #
+        #  Starts EM loop and sets up subscription callbacks for workers. 
+        #
+        def listen
+          EM.run do
+            connect do
+              routes.each do |route|
+                @client.subscribe(route) do |args|
+                  begin
+                    run(route, args)
+                  rescue
+                    logger.error("EventmachineSubscriber listen error on #{route}: #{$!}")
+                  end
+                end
+              end
+            end
+          end
+        end
+                
+        def stop
+          EM.stop if EM.reactor_running?
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/looped_subscriber.rb

@@ -0,0 +1,38 @@
+require 'workling/remote/invokers/base'
+
+#
+#  Subscribes the workers to the correct queues. 
+# 
+module Workling
+  module Remote
+    module Invokers
+      class LoopedSubscriber < Workling::Remote::Invokers::Base
+
+        def initialize(routing, client_class)
+          super
+        end
+
+        #
+        #  Starts EM loop and sets up subscription callbacks for workers. 
+        #
+        def listen
+          connect do
+            routes.each do |route|
+              @client.subscribe(route) do |args|
+                run(route, args)
+              end
+            end
+
+            loop do
+              sleep 1
+            end
+          end
+        end
+
+        def stop
+          
+        end
+      end
+    end
+  end
+end

data/lib/workling/remote/invokers/thread_pool_poller.rb

@@ -0,0 +1,169 @@
+require 'workling/remote/invokers/base'
+
+require 'thread'
+require 'mutex_m'
+
+#
+# A polling invoker that executes the jobs using a thread pool.
+#
+# This invoker was designed for long running tasks and Rails 2.2. It is expected
+# that each worker will manage it's own database connections using ActiveRecord::Base.connection_pool.
+#
+# This implementation isn't using most of the features provided by Invokers::Base
+# because of it's assumptions about database connections
+#
+# Example:
+#   user = nil
+#   ActiveRecord::Base.connection_pool.with_connection do
+#     user = User.find(options[:id])
+#   end
+#
+#   # Do long running stuff here
+#   user.wait_until_birthday
+#   user.buy_gift 'BRAWNDO!'
+#
+#   ActiveRecord::Base.connection_pool.with_connection do
+#     user.save
+#   end
+#
+module Workling
+  module Remote
+    module Invokers
+      class ThreadPoolPoller < Workling::Remote::Invokers::Base
+        attr_reader :sleep_time, :reset_time, :pool_capacity
+
+        def initialize(routing, client_class)
+          @routing = routing
+          @client_class = client_class
+
+          # Grab settings out of the config file
+          @sleep_time = (Workling.config[:sleep_time] || 2).to_f
+          @reset_time = (Workling.config[:reset_time] || 30).to_f
+
+          # Pool of polling threads
+          @pollers = []
+          @pollers.extend(Mutex_m)
+
+          # Pool of worker threads
+          @workers = []
+          @workers.extend(Mutex_m)
+
+          # Connection to the job queue
+          @pool_capacity = (Workling.config[:pool_size] || 25).to_i
+        end
+
+        # Start up the checking for items on the queue. Will block until stop is called and all pollers
+        # and workers have finished execution.
+        def listen
+          logger.info("Starting ThreadPoolPoller...")
+
+          # Determine which queues need checking
+          Workling::Discovery.discovered.map do |klass|
+            @pollers.synchronize do
+              # Polls the backing queue for jobs to be done
+              @pollers << Thread.new do
+                poller_thread(@routing.queue_names_routing_class(klass))
+              end
+            end
+          end
+
+          # Wait for the poller and all outstanding workers to finish.
+          #
+          # This is a little tricky because we're doing some synchronization on pollers... but
+          # the list of pollers is never modified after being setup above.
+          @pollers.synchronize { @pollers.dup }.each { |p| p.join }
+          @pollers.synchronize { @pollers.clear }
+          logger.info("Pollers have all finished")
+
+          @workers.synchronize { @workers.dup }.each { |w| w.join }
+          logger.info("Worker threads have all finished")
+        end
+
+        # Instructs the thread pool poller to stop checking for new jobs on the backing queue.
+        def stop
+          logger.info("Stopping thread pool invoker pollers and workers...")
+          @pollers.synchronize { @pollers.each { |p| p[:shutdown] = true } }
+        end
+
+        # Set pool_size in workling config to adjust the maximum number of threads in the pool
+        def workers_available?
+          worker_threads < @pool_capacity
+        end
+
+        # Number of correctly active worker threads
+        def worker_threads
+          @workers.synchronize { @workers.size }
+        end
+
+        # Number of currently active polling threads
+        def poller_threads
+          @pollers.synchronize { @pollers.size }
+        end
+
+        private
+
+        def poller_thread(queues)
+          # Make sure queues is an array
+          queues = [queues].flatten!
+
+          # Connect our client to the backing queue
+          client = @client_class.new
+          client.connect
+          logger.info("** Starting client #{ client.class } for #{ queues.inspect }") if logger.info?
+
+          # Poll each queue for new items
+          while(!Thread.current[:shutdown]) do
+            # Check each queue for a job posting
+            queues.each do |queue|
+              break if Thread.current[:shutdown]
+
+              begin
+                # Take a job off the queue and execute it in a new worker thread.
+                #
+                # Don't pop any jobs off the backing queue if the thread pool
+                # is full. This keeps them on the master queue so other instances
+                # can process them.
+                while(workers_available? && (options = client.retrieve(queue)))
+                  logger.debug("#{queue} received job #{ options.inspect }") if logger.debug?
+
+                  @workers.synchronize do
+                    @workers << Thread.new do
+                      begin
+                        # Execute the job
+                        run(queue, options)
+                      rescue Exception => e
+                        # Log the exception since there isn't much else we can do about it at this point
+                        logger.error(e) if logger.error?
+                      ensure
+                        # Make sure the current thread's connection gets released
+                        if(ActiveRecord::Base.connection_pool)
+                          ActiveRecord::Base.connection_pool.release_connection
+                        end
+
+                        # Remove this thread from the list of active workers
+                        @workers.synchronize do
+                          @workers.delete(Thread.current)
+                        end
+                      end
+                    end
+                  end
+
+                  # Break out of the checks early if shutdown was called
+                  break if Thread.current[:shutdown]
+                end
+              rescue Workling::WorklingError => e
+                logger.error("FAILED to connect with queue #{ queue }: #{ e } }") if logger.error?
+                sleep(@reset_time)
+
+                # FIXME: This will _definitely_ blow up with AMQP since there is no reset call
+                client.reset
+              end
+            end
+
+            sleep(@sleep_time) unless Thread.current[:shutdown]
+          end
+        end
+      end
+    end
+  end
+end