elecnix-workling 0.4.2 → 0.4.9

data/lib/workling/base.rb

@@ -1,71 +0,0 @@
-#require 'struct'
-
-#
-#  All worker classes must inherit from this class, and be saved in app/workers. 
-# 
-#  The Worker lifecycle: 
-#    The Worker is loaded once, at which point the instance method 'create' is called. 
-#
-#  Invoking Workers: 
-#    Calling async_my_method on the worker class will trigger background work.
-#    This means that the loaded Worker instance will receive a call to the method
-#    my_method(:uid => "thisjobsuid2348732947923"). 
-#
-#    The Worker method must have a single hash argument. Note that the job :uid will
-#    be merged into the hash. 
-#
-module Workling
-  class Base
-    cattr_accessor :logger
-    @@logger ||= ::RAILS_DEFAULT_LOGGER
-    
-    def self.inherited(subclass)
-      Workling::Discovery.discovered << subclass
-    end
-    
-    def initialize
-      super
-      
-      create
-    end
-
-    # Put worker initialization code in here. This is good for restarting jobs that
-    # were interrupted.
-    def create
-    end
-    
-    # takes care of suppressing remote errors but raising Workling::WorklingNotFoundError
-    # where appropriate. swallow workling exceptions so that everything behaves like remote code.
-    # otherwise StarlingRunner and SpawnRunner would behave too differently to NotRemoteRunner.
-    def dispatch_to_worker_method(method, options = {})
-      begin
-        options = default_options.merge(options)
-        self.send(method, options)
-      rescue Workling::WorklingError => e
-        raise e
-      rescue Exception => e
-        raise e if e.kind_of? Workling::WorklingError
-        logger.error "WORKLING ERROR: runner could not invoke #{ self.class }:#{ method } with #{ options.inspect }. error was: #{ e.class }:#{ e.message }\n #{ e.backtrace.join("\n") }"
-
-        on_error(e) if respond_to?(:on_error)
-
-        # reraise after logging. the exception really can't go anywhere in many cases. (spawn traps the exception)
-        raise e if Workling.raise_exceptions?
-      end
-    end    
-
-    # supply default_options as a hash in classes that inherit Workling::Base
-    def default_options
-      {}
-    end
-  
-    # thanks to blaine cook for this suggestion.
-    def self.method_missing(method, *args, &block)
-      if method.to_s =~ /^asynch?_(.*)/
-        Workling::Remote.run(self.to_s.dasherize, $1, *args)
-      else
-        super
-      end
-    end
-  end
-end

data/lib/workling/clients/amqp_client.rb

@@ -1,56 +0,0 @@
-require 'workling/clients/base'
-Workling.try_load_an_amqp_client
-
-#
-#  An Ampq client
-#
-module Workling
-  module Clients
-    class AmqpClient < Workling::Clients::Base
-            
-      # starts the client. 
-      def connect
-        begin
-          @options = (Workling.config[:amqp_options] || {}).symbolize_keys
-          # host and port only needed for AMQP.start
-          host, port = Workling.config[:listens_on].split(':', 2)
-          start_opts = {:host => host || 'localhost', :port => (port || 5672).to_i}.merge(@options)
-          @amq = MQ.new(AMQP.start(start_opts))
-        rescue
-          raise WorklingError.new("couldn't start amq client. if you're running this in a server environment, then make sure the server is evented (ie use thin or evented mongrel, not normal mongrel.): #{$!}")
-        end
-      end
-      
-      # no need for explicit closing. when the event loop
-      # terminates, the connection is closed anyway. 
-      def close; true; end
-      
-      # Decide which method of marshalling to use:
-      def cast_value(value, origin)
-        case Workling.config[:ymj]
-        when 'marshal'
-          @cast_value = origin == :request ? Marshal.dump(value) : Marshal.load(value)
-        when 'yaml'
-          @cast_value = origin == :request ? YAML.dump(value) : YAML.load(value)
-        else 
-          @cast_value = origin == :request ? Marshal.dump(value) : Marshal.load(value)
-        end
-        @cast_value
-      end
-      
-      # subscribe to a queue
-      def subscribe(key)
-        @amq.queue(key, @options).subscribe(@options) do |value|
-          yield YAML.load(value, :subscribe)
-        end
-      end
-      
-      # request and retrieve work
-      def retrieve(key); @amq.queue(key, @options); end
-      def request(key, value)
-        logger.info("> publishing to #{key}: #{value.inspect}")
-        @amq.queue(key, @options).publish(cast_value(value, :request), @options)
-      end
-    end
-  end
-end

data/lib/workling/clients/base.rb

@@ -1,57 +0,0 @@
-#
-#  Clients are responsible for communicating with a job broker (ie connecting to starling or rabbitmq.)
-#
-#  Clients are used to request jobs on a broker, get results for a job from a broker, and subscribe to results
-#  from a specific type of job. 
-#
-module Workling
-  module Clients
-    class Base
-
-      # returns the Workling::Base.logger
-      def logger; Workling::Base.logger; end
-
-      #
-      #  Requests a job on the broker.
-      #
-      #      work_type: 
-      #      arguments: the argument to the worker method
-      #
-      def request(work_type, arguments)
-        raise NotImplementedError.new("Implement request(work_type, arguments) in your client. ")
-      end    
-
-      #
-      #  Gets job results off a job broker. Returns nil if there are no results. 
-      #
-      #      worker_uid: the uid returned by workling when the work was dispatched
-      #
-      def retrieve(work_uid)
-        raise NotImplementedError.new("Implement retrieve(work_uid) in your client. ")
-      end
-      
-      #
-      #  Subscribe to job results in a job broker.
-      #
-      #      worker_type: 
-      #
-      def subscribe(work_type)
-        raise NotImplementedError.new("Implement subscribe(work_type) in your client. ")
-      end
-      
-      #
-      #  Opens a connection to the job broker.
-      #
-      def connect
-        raise NotImplementedError.new("Implement connect() in your client. ")
-      end
-      
-      #
-      #  Closes the connection to the job broker. 
-      #
-      def close
-        raise NotImplementedError.new("Implement close() in your client. ")
-      end
-    end
-  end
-end

data/lib/workling/clients/memcache_queue_client.rb

@@ -1,83 +0,0 @@
-require 'workling/clients/base'
-require 'memcache'
-
-#
-#  This client can be used for all Queue Servers that speak Memcached, such as Starling. 
-#
-#  Wrapper for the memcache connection. The connection is made using fiveruns-memcache-client, 
-#  or memcache-client, if this is not available. See the README for a discussion of the memcache 
-#  clients. 
-#
-#  method_missing delegates all messages through to the underlying memcache connection. 
-#
-module Workling
-  module Clients
-    class MemcacheQueueClient < Workling::Clients::Base
-      
-      # the class with which the connection is instantiated
-      cattr_accessor :memcache_client_class
-      @@memcache_client_class ||= ::MemCache
-      
-      # the url with which the memcache client expects to reach starling
-      attr_accessor :queueserver_urls
-      
-      # the memcache connection object
-      attr_accessor :connection
-      
-      #
-      #  the client attempts to connect to queueserver using the configuration options found in 
-      #
-      #      Workling.config. this can be configured in config/workling.yml. 
-      #
-      #  the initialization code will raise an exception if memcache-client cannot connect 
-      #  to queueserver.
-      #
-      def connect
-        @queueserver_urls = Workling.config[:listens_on].split(',').map { |url| url ? url.strip : url }
-        options = [@queueserver_urls, Workling.config[:memcache_options]].compact
-        self.connection = MemcacheQueueClient.memcache_client_class.new(*options)
-        
-        raise_unless_connected!
-      end
-      
-      # closes the memcache connection
-      def close
-        self.connection.flush_all
-        self.connection.reset
-      end
-
-      # implements the client job request and retrieval 
-      def request(key, value)
-        set(key, value)
-      end
-      
-      def retrieve(key)
-        begin
-          get(key)
-        rescue MemCache::MemCacheError => e
-          # failed to enqueue, raise a workling error so that it propagates upwards
-          raise Workling::WorklingError.new("#{e.class.to_s} - #{e.message}")        
-        end
-      end
-            
-      private
-        # make sure we can actually connect to queueserver on the given port
-        def raise_unless_connected!
-          begin 
-            self.connection.stats
-          rescue
-            raise Workling::QueueserverNotFoundError.new
-          end
-        end
-        
-        # delegates directly through to the memcache connection. 
-        def method_missing(method, *args)
-          begin
-            self.connection.send(method, *args)
-          rescue MemCache::MemCacheError => e
-            raise Workling::WorklingConnectionError.new("#{e.class.to_s} - #{e.message}")        
-          end
-        end
-    end
-  end
-end

data/lib/workling/discovery.rb

@@ -1,14 +0,0 @@
-#
-#  Discovery is responsible for loading workers in app/workers. 
-#
-module Workling
-  class Discovery
-    cattr_accessor :discovered
-    @@discovered = []
-    
-    # requires worklings so that they are added to routing. 
-    def self.discover!
-      Workling.load_path.map{|p| Dir.glob("#{p}/**/*.rb")}.flatten.each{|wling| require wling}
-    end
-  end
-end

data/lib/workling/remote.rb

@@ -1,42 +0,0 @@
-require "workling/remote/runners/not_remote_runner"
-require "workling/remote/runners/spawn_runner"
-require "workling/remote/runners/starling_runner"
-require "workling/remote/runners/backgroundjob_runner"
-
-require 'digest/md5'
-
-#
-#  Scoping Module for Runners. 
-#
-module Workling
-  module Remote
-    
-    # set the desired runner here. this is initialized with Workling.default_runner. 
-    mattr_accessor :dispatcher
-    
-    # set the desired invoker. this class grabs work from the job broker and executes it. 
-    mattr_accessor :invoker
-    @@invoker ||= Workling::Remote::Invokers::ThreadedPoller
-    
-    # retrieve the dispatcher or instantiate it using the defaults
-    def self.dispatcher
-      @@dispatcher ||= Workling.default_runner
-    end
-    
-    # generates a unique identifier for this particular job. 
-    def self.generate_uid(clazz, method)
-      uid = ::Digest::MD5.hexdigest("#{ clazz }:#{ method }:#{ rand(1 << 64) }:#{ Time.now }")
-      "#{ clazz.to_s.tableize }/#{ method }/#{ uid }".split("/").join(":")
-    end
-    
-    # dispatches to a workling. writes the :uid for this work into the options hash, so make 
-    # sure you pass in a hash if you want write to a return store in your workling.
-    def self.run(clazz, method, options = {})
-      uid = Workling::Remote.generate_uid(clazz, method)
-      options[:uid] = uid if options.kind_of?(Hash) && !options[:uid]
-      Workling.find(clazz, method) # this line raises a WorklingError if the method does not exist. 
-      dispatcher.run(clazz, method, options)
-      uid
-    end
-  end
-end

data/lib/workling/remote/invokers/base.rb

@@ -1,124 +0,0 @@
-#
-#  Invokers are responsible for 
-#
-#      1. grabbing work off a job broker (such as a starling or rabbitmq server).
-#      2. routing (mapping) that work onto the correct worker method. 
-#      3.invoking the worker method, passing any arguments that came off the broker.
-#
-#   Invokers should implement their own concurrency strategies. For example, 
-#   The there is a ThreadedPoller which starts a thread for each Worker class.
-#
-#   This base Invoker class defines the methods an Invoker needs to implement. 
-#  
-module Workling
-  module Remote
-    module Invokers
-      class Base
-        
-        attr_accessor :sleep_time, :reset_time
-        
-        #
-        #  call up with super in the subclass constructor.
-        #
-        def initialize(routing, client_class)
-          @routing = routing
-          @client_class = client_class
-          @sleep_time = Workling.config[:sleep_time] || 2
-          @reset_time = Workling.config[:reset_time] || 30
-          @@mutex ||= Mutex.new
-        end
-        
-        #
-        #  Starts main Invoker Loop. The invoker runs until stop() is called. 
-        #
-        def listen
-          raise NotImplementedError.new("Implement listen() in your Invoker. ")
-        end        
-        
-        #
-        #  Gracefully stops the Invoker. The currently executing Jobs should be allowed
-        #  to finish. 
-        #
-        def stop
-          raise NotImplementedError.new("Implement stop() in your Invoker. ")
-        end
-        
-        # 
-        #  Runs the worker method, given
-        #
-        #      type: the worker route
-        #      args: the arguments to be passed into the worker method.
-        #
-        def run(type, args)
-          worker = @routing[type]
-          method = @routing.method_name(type)
-          worker.dispatch_to_worker_method(method, args)
-        end
-        
-        # returns the Workling::Base.logger
-        def logger; Workling::Base.logger; end
-        
-        protected
-        
-          # handle opening and closing of client. pass code block to this method. 
-          def connect
-            @client = @client_class.new
-            @client.connect
-            
-            begin
-              yield
-            ensure
-              @client.close
-              ActiveRecord::Base.verify_active_connections!
-            end
-          end
-
-          #
-          #  Loops through the available routes, yielding for each route. 
-          #  This continues until @shutdown is set on this instance. 
-          #
-          def loop_routes
-            while(!@shutdown) do
-              ensure_activerecord_connection
-              
-              routes.each do |route|
-                break if @shutdown
-                yield route
-              end
-              
-              sleep self.sleep_time
-            end
-          end
-
-          #
-          #  Returns the complete set of active routes
-          #
-          def routes
-            @active_routes ||= Workling::Discovery.discovered.map { |clazz| @routing.queue_names_routing_class(clazz) }.flatten
-          end
-          
-          # Thanks for this Brent! 
-          #
-          #     ...Just a heads up, due to how rails’ MySQL adapter handles this  
-          #     call ‘ActiveRecord::Base.connection.active?’, you’ll need 
-          #     to wrap the code that checks for a connection in in a mutex.
-          #
-          #     ....I noticed this while working with a multi-core machine that 
-          #     was spawning multiple workling threads. Some of my workling 
-          #     threads would hit serious issues at this block of code without 
-          #     the mutex.            
-          #
-          def ensure_activerecord_connection
-            @@mutex.synchronize do 
-              unless ActiveRecord::Base.connection.active?  # Keep MySQL connection alive
-                unless ActiveRecord::Base.connection.reconnect!
-                  logger.fatal("Failed - Database not available!")
-                  break
-                end
-              end
-            end            
-          end
-      end
-    end
-  end
-end