background_queue 0.2.0

data/lib/background_queue/server_lib/worker_thread.rb

@@ -0,0 +1,70 @@
+
+module BackgroundQueue::ServerLib
+  #A Thread that processes the task queue
+  class WorkerThread
+  
+    def initialize(server)
+      @server = server
+    end
+      
+    def get_next_task
+      task = nil
+      while @server.running? && task.nil?
+        task = @server.task_queue.next_task
+      end
+      task = nil if task.nil? || !@server.running? #if the server isnt running, dont continue to process a task
+      task
+    end
+    
+    def build_client
+      BackgroundQueue::ServerLib::WorkerClient.new(@server)
+    end
+    
+    def call_worker(task)
+      @server.change_stat(:tasks, -1)
+      @server.change_stat(:running, 1)
+      @server.logger.debug("calling worker for task #{task.id}")
+      error_count = 0
+      while @server.running?
+        worker = @server.workers.get_next_worker
+        if worker.nil?
+          Kernel.sleep(1) unless !@server.running?
+        else
+          client = build_client
+          if client.send_request(worker, task, @server.config.secret)
+            @server.logger.debug("called worker for task #{task.id}")
+            @server.workers.finish_using_worker(worker, true)
+            @server.task_queue.finish_task(task)
+            @server.change_stat(:running, -1)
+            @server.change_stat(:run_tasks, 1)
+            
+            return true
+          else
+            @server.logger.debug("failed calling worker for task #{task.id}")
+            @server.workers.finish_using_worker(worker, false)
+            error_count += 1
+            if error_count > 5
+              @server.logger.debug("error count exceeded for task #{task.id}, returning to queue")
+              @server.task_queue.finish_task(task)
+              @server.task_queue.add_task(task)
+              return true
+            end
+          end
+        end
+      end
+      #if we get here the server stopped before we could do the task... put it back so it can be saved to disk...
+      @server.logger.debug("returning task #{task.id} to queue because the server has stopped")
+      @server.task_queue.finish_task(task)
+      @server.task_queue.add_task(task)
+      
+      false
+    end
+    
+    def run
+      while @server.running?
+        task = get_next_task
+        call_worker(task) unless task.nil?
+      end
+    end
+  end
+end

data/lib/background_queue/utils.rb

@@ -0,0 +1,40 @@
+module BackgroundQueue
+  #Utility Module
+  module Utils
+    
+    #gets an entry from a hash regardless if the key is a string or symbol
+    def self.get_hash_entry(hash, key)
+      if hash.has_key?(key)
+        hash[key]
+      elsif key.kind_of?(String)
+        hash[key.intern]
+      else
+        hash[key.to_s]
+      end
+    end
+    
+    #class that wraps a hash so it can be accessed by key or symbol
+    class AnyKeyHash
+      #the wrapped hash
+      attr_accessor :hash
+      
+      #wrap a hash
+      def initialize(hash)
+        @hash = hash
+      end
+      
+      #get an entry by string or symbol
+      def [] (key)
+        BackgroundQueue::Utils.get_hash_entry(@hash, key)
+      end
+      
+      def []=(key, value)
+        @hash[key] = value
+      end
+      
+      def to_json(dummy=true)
+        @hash.to_json
+      end
+    end
+  end
+end

data/lib/background_queue/worker/base.rb

@@ -0,0 +1,46 @@
+module BackgroundQueue::Worker
+  #the base class of workers
+  class Base
+  
+    def initialize
+      @environment = nil
+    end
+    
+    def set_environment(env)
+      #puts "set_environment=#{env}"
+      @environment = env
+    end
+    
+    def environment
+      @environment
+    end
+    
+    #update the progress of the currently running task
+    def set_progress(caption, percent)
+      #puts self
+      #puts "env=#{self.environment}"
+      self.environment.send_data({:caption=>caption, :percent=>percent}.to_json)
+    end
+    
+    #add meta data to the progress
+    #key: :notice, :warning, :error, :meta
+    #value: :notice/:warning/:error : String, :meta : any json compatible object
+    def add_progress_meta(key, value)
+      self.environment.send_data({:meta=>{key=>value}}.to_json)
+    end
+    
+    #virtual function: called to process a worker request
+    def run
+      raise "run() Not Implemented on worker #{self.class.name}"
+    end
+    
+    def params
+      self.environment.params
+    end
+    
+    def logger
+      self.environment.logger
+    end
+    
+  end
+end

data/lib/background_queue/worker/calling.rb

@@ -0,0 +1,59 @@
+module BackgroundQueue::Worker
+  #this module is mixed into the controller that will service the worker calls.
+  #the action that will recive the worker calls should only call run_worker
+  module Calling
+  
+    #call this method from within the controller action that receives the worker calls. 
+    #pass the shared secret in to validate the call: this should equal the secret in the server config.
+    def run_worker
+      return unless check_secret()
+      worker = nil
+      env = nil
+      begin
+        #setup worker environment
+        env = init_environment
+        #get worker
+        worker = BackgroundQueue::Worker::WorkerLoader.get_worker(env.worker)
+        worker.set_environment(env)
+      rescue Exception=>e
+        logger.error("Error initializing worker: #{e.message}")
+        logger.debug(e.backtrace.join("\n"))
+        render :text=>"Error initializing worker: #{e.message}", :status=>500
+        raise e
+      end
+      #puts worker
+      #call worker
+      call_worker(worker, env)
+      true
+
+    end
+    
+    def init_environment
+      env = BackgroundQueue::Worker::Environment.new
+      env.init_from_controller(self)
+      env
+    end
+    
+    def call_worker(worker, env)
+      headers['X-Accel-Buffering'] = 'no' #passenger standalone uses nginx. This will turn buffering off in nginx
+      render :text => lambda { |response,output| 
+        env.set_output(output)
+        begin
+          worker.run
+        rescue Exception=>e
+          logger.error("Error calling worker: #{e.message}")
+          logger.error(e.backtrace.join("\n"))
+        ensure
+          worker.set_environment(nil)
+        end
+      }, :type=>"text/text"
+    end
+    
+    def check_secret
+      return true if params[:auth] == BackgroundQueue::Worker::Config.secret
+      render :text=>"Invalid auth (#{params[:auth]})", :status=>401
+      false
+    end
+    
+  end
+end

data/lib/background_queue/worker/config.rb

@@ -0,0 +1,35 @@
+module BackgroundQueue::Worker
+  class Config
+      
+    def self.secret=(auth)
+      @@secret = auth
+    end
+    
+    def self.secret
+      @@secret
+    end
+    
+    def self.worker_path=(path)
+      @@worker_path = path
+    end
+    
+    def self.worker_path
+      if @@worker_path.nil?
+        default_worker_path
+      else
+        @@worker_path
+      end
+    end
+    
+    def self.default_worker_path
+      if defined?(RAILS_ROOT)
+        File.join(RAILS_ROOT, "lib", "bgq_workers")
+      elsif defined?(Rails) 
+        File.join(Rails.root, "lib", "bgq_workers")
+      else
+        raise "You must specify the BackgroundQueue::Worker::Config.worker_path"
+      end
+    end
+    
+  end
+end

data/lib/background_queue/worker/environment.rb

@@ -0,0 +1,70 @@
+module BackgroundQueue::Worker
+  #holds the params, controller and response
+  class Environment
+    attr_reader :params
+    attr_reader :owner_id 
+    attr_reader :job_id
+    attr_reader :task_id
+    attr_reader :priority
+    attr_reader :owner_id
+    attr_reader :worker
+    attr_reader :logger
+    
+    attr_reader :server_address
+    
+    attr_reader :output
+    
+    def initialize
+      @params = {}
+    end
+    
+    def init_from_controller(controller)
+      @controller = controller
+      @logger = controller.logger
+      init_params(controller.params)
+      init_server_address(controller)
+    end
+    
+    def init_params(controller_params)
+      hash_data = nil
+      begin
+        hash_data = JSON.load(controller_params[:task])
+      rescue Exception=>e
+        raise "Invalid data format (should be json) when loading task from buffer: #{e.message}"
+      end
+      raise 'Invalid json root object (should be hash)' unless hash_data.kind_of?(Hash)
+      
+      @params = BackgroundQueue::Utils::AnyKeyHash.new(hash_data['params'])
+      @owner_id = hash_data['owner_id']
+      @job_id = hash_data['job_id']
+      @task_id = hash_data['id']
+      @priority = hash_data['priority']
+      @worker = hash_data['worker']
+    end
+    
+    def set_output(out)
+      @output = out
+    end
+    
+    def send_data(data)
+      @output.write("#{data}\n")
+      @output.flush
+    end
+    
+    def init_server_address(controller)
+      @server_address = BackgroundQueue::Worker::Environment::Server.new(controller.request.remote_ip, controller.params[:server_port])
+    end
+    
+    class Server
+      attr_accessor :host
+      attr_accessor :port
+      
+      def initialize(host, port)
+        @host = host
+        @port = port
+      end
+      
+    end
+    
+  end
+end

data/lib/background_queue/worker/worker_loader.rb

@@ -0,0 +1,94 @@
+module BackgroundQueue::Worker
+  #looks after loading and caching worker classes.
+  #worker classes sit in BackgroundQueue::Worker::Config.worker_path directory
+  class WorkerLoader
+    
+    def initialize
+      @worker_entries = {}
+    end
+    
+    def self.get_worker(worker_name)
+      @@worker_loader ||= BackgroundQueue::Worker::WorkerLoader.new
+      @@worker_loader.get_worker(worker_name)
+    end
+  
+    def get_worker(worker_name)
+      worker_entry = get(worker_name)
+      if worker_entry.nil?
+        worker_entry = load_worker(worker_name)
+        set(worker_entry)
+      else
+        reload_if_updated(worker_entry)
+      end
+      worker_entry.worker
+    end
+
+    def load_worker(worker_name)
+      path = worker_path(worker_name)
+      load_file(path)
+      datestamp = File.mtime(path)
+      worker = load_class(worker_name, path)
+      WorkerEntry.new(worker, path, datestamp, worker_name)
+    end
+    
+    def load_file(path)
+      load(path)
+    end
+    
+    def load_class(worker_name, path)
+      class_name = worker_class_name(worker_name)
+      begin
+        eval("#{class_name}.new")
+      rescue NameError=>e
+        raise "#{path} did not define #{class_name}"
+      end
+    end
+   
+    def worker_class_name(worker_name)
+      worker_name.split('_').collect!{ |w| w.capitalize }.join
+    end
+    
+    def worker_path(worker_name)
+      File.join(BackgroundQueue::Worker::Config.worker_path, "#{worker_name}.rb")
+    end
+    
+    def reload_if_updated(worker_entry)
+      ds = File.mtime(worker_entry.path)
+      if ds != worker_entry.datestamp
+        load_file(worker_entry.path)
+        worker_entry.worker = load_class(worker_entry.name)
+        worker_entry.datestamp = ds
+      end
+    end
+    
+    
+    
+    class WorkerEntry
+      
+      attr_accessor :worker
+      attr_accessor :path
+      attr_accessor :datestamp
+      attr_accessor :name
+      
+      def initialize(worker, path, datestamp, name)
+        @worker = worker
+        @path = path
+        @datestamp = datestamp
+        @name = name
+      end
+      
+    end
+    
+    private
+    
+    def set(worker_entry)
+      @worker_entries[worker_entry.name] = worker_entry
+    end
+    
+    def get(worker_name)
+      @worker_entries[worker_name]
+    end
+    
+    
+  end
+end

data/lib/background_queue_server.rb

@@ -0,0 +1,21 @@
+require "background_queue/utils"
+require "background_queue/config"
+require "background_queue/server_lib/lru"
+require "background_queue/server_lib/config"
+require "background_queue/server_lib/priority_queue"
+require "background_queue/server_lib/queue_registry"
+require "background_queue/server_lib/thread_manager"
+require "background_queue/server_lib/balanced_queue"
+require "background_queue/server_lib/owner"
+require "background_queue/server_lib/job"
+require "background_queue/server_lib/job_registry"
+require "background_queue/server_lib/task"
+require "background_queue/server_lib/task_registry"
+require "background_queue/server_lib/sorted_workers"
+require "background_queue/server_lib/worker"
+require "background_queue/server_lib/worker_client"
+require "background_queue/server_lib/worker_balancer"
+require "background_queue/server_lib/worker_thread"
+require "background_queue/server_lib/server"
+require "background_queue/server_lib/event_server"
+require "background_queue/server_lib/event_connection"