background_queue 0.2.0

data/lib/background_queue/client_lib/command.rb

@@ -0,0 +1,36 @@
+require 'json'
+
+module BackgroundQueue::ClientLib
+
+  #store a command and all its parameters as a hash to be serialized when passing to the server.
+  module Command
+  
+    #create an 'add task' command
+    def self.add_task_command(worker, owner_id, job_id, task_id, priority, task_parameters={}, options={} )
+      BackgroundQueue::Command.new(:add_task, options, :worker=>worker, :owner_id=>owner_id, :job_id=>job_id, :task_id=>task_id, :priority=>priority, :params=>task_parameters)
+    end
+    
+    #create an 'add tasks' command
+    def self.add_tasks_command(worker, owner_id, job_id, tasks, priority, shared_parameters={}, options={} )
+      raise BackgroundQueue::InvalidCommand, "No Tasks In List" if tasks.nil? || tasks.length == 0
+      BackgroundQueue::Command.new(:add_tasks, options, :worker=>worker, :owner_id=>owner_id, :job_id=>job_id, :tasks=>tasks, :priority=>priority, :shared_parameters=>shared_parameters)
+    end
+    
+    #create an 'get_status' command
+    def self.get_status_command(job_id, options={} )
+      BackgroundQueue::Command.new(:get_status, options, {:job_id=>job_id})
+    end
+    
+    #create an 'stats' command
+    def self.stats_command(options={})
+      BackgroundQueue::Command.new(:stats, options, {})
+    end
+    
+    #create a 'remove tasks' command
+    #is this needed?
+    #def self.remove_tasks_command(tasks, options={})
+    #  raise BackgroundQueue::InvalidCommand, "No Tasks In List" if tasks.nil? || tasks.length == 0
+    #  BackgroundQueue::Command.new(:remove_tasks, options, {:tasks=>tasks})
+    #end
+  end
+end

data/lib/background_queue/client_lib/config.rb

@@ -0,0 +1,109 @@
+module BackgroundQueue::ClientLib
+
+  #The client configuration which is stored as a YAML file containing a root key for each environments configuration, much like database.yml.
+  #
+  #Example
+  #=======
+  #
+  #   development:
+  #     server:
+  #       host: 127.0.0.1
+  #       port: 3000
+  #     memcache: 127.0.0.1:9999
+  #   production:
+  #     server: 
+  #       host: 192.168.3.56
+  #       port: 3000
+  #     failover: 
+  #       -
+  #         host: 192.168.3.57
+  #         port: 3000
+  #       -
+  #         host: 192.168.3.58
+  #     memcache: 192.168.3.1:9999, 192.168.3.3:9999
+  class Config < BackgroundQueue::Config
+
+    #the primary {BackgroundQueue::Config::Server}
+    attr_reader :server
+    #an array of failover {BackgroundQueue::Config::Server}
+    attr_reader :failover
+    #an array of Strings defining memcache server address
+    attr_reader :memcache
+    
+    #load the configration using a hash just containing the environment
+    def self.load_hash(env_config, path)
+      BackgroundQueue::ClientLib::Config.new(
+        build_primary_server_entry(env_config, path),
+        build_failover_server_entries(env_config, path),
+        build_memcache_array(env_config, path)
+      )
+    end
+    
+    class << self
+      private
+      
+      def build_primary_server_entry(env_config, path)
+        server_entry = BackgroundQueue::Utils.get_hash_entry(env_config, :server)
+        if server_entry
+          begin
+            BackgroundQueue::ClientLib::Config::Server.new(server_entry)
+          rescue Exception=>e
+            raise BackgroundQueue::LoadError, "Error loading 'server' entry from background queue configuration file #{full_path(path)}: #{e.message}"
+          end
+        else
+          raise BackgroundQueue::LoadError, "Missing 'server' entry in background queue configuration file #{full_path(path)}"
+        end
+      end
+      
+      def build_failover_server_entries(env_config, path)
+        entries = []
+        failover_entry = BackgroundQueue::Utils.get_hash_entry(env_config, :failover)
+        if failover_entry && failover_entry.kind_of?(Array)
+          failover_entry.each_with_index do |entry, index|
+            begin
+              entries << BackgroundQueue::ClientLib::Config::Server.new(entry)
+            rescue Exception=>e
+              raise BackgroundQueue::LoadError, "Error loading 'failover' entry (#{index + 1}) from background queue configuration file #{full_path(path)}: #{e.message}"
+            end
+          end
+        elsif failover_entry
+          raise BackgroundQueue::LoadError, "Error loading 'failover' entries configuration file #{full_path(path)}: invalid data type (#{failover_entry.class.name}), expecting Array"
+        end
+        entries
+      end
+      
+    end
+    
+    
+    #do not call this directly, use a load_* method
+    def initialize(server, failover, memcache)
+      @server = server
+      @failover = failover
+      @memcache = memcache
+    end
+    
+    #A server entry in the configuration
+    class Server
+      
+      attr_reader :host
+      attr_reader :port
+      
+      def initialize(config_entry)
+        if config_entry.kind_of?(Hash)
+          @host = BackgroundQueue::Utils.get_hash_entry(config_entry, :host)
+          raise BackgroundQueue::LoadError, "Missing 'host' configuration entry" if @host.nil?
+        
+          @port = BackgroundQueue::Utils.get_hash_entry(config_entry, :port)
+          if @port
+            @port = @port.to_i
+          else
+            @port = BackgroundQueue::Config::DEFAULT_PORT 
+          end
+        else
+          raise BackgroundQueue::LoadError, "Invalid data type (#{config_entry.class.name}), expecting Hash"
+        end
+      end
+    end
+  end
+  
+end

data/lib/background_queue/client_lib/connection.rb

@@ -0,0 +1,105 @@
+require 'timeout'
+
+module BackgroundQueue::ClientLib
+  #A connection to a backend queue server
+  #Handles sending the command to the server and receiving the reply
+  #For now connections are not pooled/reused
+  class Connection
+    def initialize(client, server)
+      @client = client
+      @server = server
+      @socket = nil
+    end
+    
+    #send a command to the server
+    def send_command(command)
+      check_connected
+      send_with_header(command.to_buf)
+      response = receive_with_header
+      BackgroundQueue::Command.from_buf(response)
+    end
+    
+    private
+    
+    def connect
+      begin
+        Timeout::timeout(3) {
+          @socket = TCPSocket.open(@server.host, @server.port)
+          true
+        }
+      rescue Timeout::Error
+        raise BackgroundQueue::ClientLib::ConnectionError, "Timeout Connecting to #{@server.host}:#{@server.port}"
+      rescue Exception=>e
+        raise BackgroundQueue::ClientLib::ConnectionError, "Error Connecting to #{@server.host}:#{@server.port}: #{e.message}"
+      end
+    end
+    
+    def check_connected
+      connect if @socket.nil?
+    end
+    
+    def send_data(data)
+      written = 0
+      to_write = data.length
+      while written < to_write do
+        begin
+          Timeout::timeout(5) {
+            amt_written = @socket.write(data)
+            written += amt_written
+            if written < to_write
+              data = data[amt_written, data.length - amt_written  ]
+            end    
+          }
+        rescue Timeout::Error
+          raise BackgroundQueue::ClientLib::ConnectionError, "Timeout Sending to #{@server.host}:#{@server.port}"
+        rescue Exception=>e
+          raise BackgroundQueue::ClientLib::ConnectionError, "Error Sending to #{@server.host}:#{@server.port}: #{e.message}"
+        end
+      end
+      true
+    end
+    
+    def self.add_header(data)
+      length = data.length
+      [1,length, data].pack("SLZ#{length}")
+    end
+    
+    def send_with_header(data)
+      send_data(BackgroundQueue::ClientLib::Connection.add_header(data))
+    end
+    
+    def receive_data(length)
+      to_read = length
+      amt_read = 0
+      sbuf = ""
+      while amt_read < to_read do
+        begin
+          Timeout::timeout(5) {
+            read_amt = to_read - amt_read
+            tbuf = @socket.recvfrom(read_amt)[0]
+            sbuf << tbuf
+            amt_read += tbuf.length
+          }
+        rescue Timeout::Error
+          raise BackgroundQueue::ClientLib::ConnectionError, "Timeout Receiving #{length} bytes from #{@server.host}:#{@server.port}"
+        rescue Exception=>e
+          raise BackgroundQueue::ClientLib::ConnectionError, "Error Receiving #{length} bytes from #{@server.host}:#{@server.port}: #{e.message}"
+        end
+      end
+      sbuf
+    end
+    
+    def receive_with_header
+      header = receive_data(6).unpack("SL")
+      receive_data(header[1])
+    end
+    
+    
+  end
+  
+  
+  #Error raised when communication failure occurs
+  class ConnectionError < Exception
+    
+  end
+end

data/lib/background_queue/client_lib/job_handle.rb

@@ -0,0 +1,19 @@
+module BackgroundQueue::ClientLib
+  #returned from add_task to describe what the job/server was added.
+  #this is becuase you can call add_task without a job_id, and not know what server was used.
+  #this is passed to get_status
+  class JobHandle
+    
+    attr_reader :owner_id
+    attr_reader :job_id
+    attr_reader :server
+    
+    def initialize(owner_id, job_id, server)
+      @owner_id = owner_id
+      @job_id = job_id
+      @server = server
+    end
+  
+  end
+
+end

data/lib/background_queue/command.rb

@@ -0,0 +1,49 @@
+require 'json'
+
+module BackgroundQueue
+
+  #store a command and all its parameters as a hash to be serialized when passing to/from the server.
+  class Command
+
+    attr_accessor :code
+    attr_accessor :options
+    attr_accessor :args
+    
+    def initialize(code, options, args)
+      @code = code
+      @options = BackgroundQueue::Utils::AnyKeyHash.new(options)
+      @args = BackgroundQueue::Utils::AnyKeyHash.new(args)
+    end
+    
+    #convert the command to a string (currently json) to get sent
+    def to_buf
+      {:c=>@code, :a=>@args.hash, :o=>@options.hash}.to_json
+    end
+    
+    #load a command from a string
+    def self.from_buf(buf)
+      hash_data = nil
+      begin
+        hash_data = JSON.load(buf)
+      rescue Exception=>e
+        raise InvalidCommand, "Invalid data format (should be json) when loading command from buffer: #{e.message}"
+      end
+      begin
+        raise "Missing 'c' (code)" if hash_data['c'].nil?
+        code = hash_data['c'].intern
+        raise "Missing 'a' (args)" if hash_data['a'].nil?
+        args = hash_data['a']
+        raise "Missing 'o' (options)" if hash_data['o'].nil?
+        options = hash_data['o']
+        BackgroundQueue::Command.new(code, options, args)
+      rescue Exception=>e
+        raise InvalidCommand, "Error loading command from buffer: #{e.message}"
+      end
+    end
+  end
+  
+  #Error raised when command is invalid
+  class InvalidCommand < Exception
+    
+  end
+end

data/lib/background_queue/config.rb

@@ -0,0 +1,118 @@
+require "erb"
+require "yaml"
+
+module BackgroundQueue
+
+  #Base class that loads configuration files for client/server
+  class Config
+    
+    #the default port that is used if the configuration does not specify a port
+    DEFAULT_PORT = 2222
+    
+    #load the configuration using a file path
+    def self.load_file(path)
+      string = get_string_from_file(path)
+      load_string(string, path)
+    end
+    
+    #load the configuration using a string that may contain ERB syntax
+    def self.load_string(string, path)
+      evaled_string = evaluate_erb(string, path)
+      load_yaml(evaled_string, path)
+    end
+    
+    #load the configuration using a string of YAML
+    def self.load_yaml(yaml_string, path)
+      all_configs = convert_yaml_to_hash(yaml_string, path)
+      env_config = extract_enviroment_entry(all_configs, path)
+      load_hash(env_config, path)
+    end
+    
+    #load the configration using a hash just containing the environment. Overridden by Client/Server class.
+    def self.load_hash(env_config, path)
+      raise "Invalid Loading of Ciguration using abstract base class. Use Server or Client subclass."
+    end
+    
+
+    class << self
+      private
+      
+      #nothing more annoying than not understanding where the library thinks path is pointing to...
+      def full_path(path)
+        if path.nil?
+          '<unknown>'
+        else
+          begin
+            File.expand_path(path)
+          rescue
+            '<unknown>'
+          end
+        end
+      end
+      
+      def get_string_from_file(path)
+        if File.exist?(path)
+          File.open(path) { |f| f.read }
+        else
+          raise BackgroundQueue::LoadError, "Failed to open background_queue configuration file at '#{full_path(path)}'"
+        end
+      end
+      
+      def evaluate_erb(string, path)
+        begin
+          message = ERB.new(string)
+          message.result
+        rescue Exception=>ex
+          raise BackgroundQueue::LoadError, "Error executing ERB for background_queue configuration file at '#{full_path(path)}': #{ex.message}"
+        end
+      end
+      
+      
+      
+      def convert_yaml_to_hash(string, path)
+        begin
+          result = YAML::load(string)
+          raise "Root of config should be a hash of environment configurations" unless result.kind_of?(Hash)
+          result
+        rescue Exception=>ex
+          raise BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at '#{full_path(path)}': #{ex.message}"
+        end
+      end
+      
+      def current_environment
+        if ENV.has_key?('RAILS_ENV')
+          ENV['RAILS_ENV']
+        elsif defined? Rails
+          Rails.env
+        end
+      end
+      
+      def extract_enviroment_entry(all_configs, path)
+        env_str = current_environment
+        if all_configs.has_key?(env_str)
+          all_configs[env_str]
+        elsif all_configs.has_key?(env_str.to_s.intern)
+          all_configs[env_str.to_s.intern]
+        else
+          raise BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at '#{full_path(path)}': missing enviroment root entry: #{env_str}"
+        end
+      end
+      
+      def build_memcache_array(env_config, path)
+        memcache_entry = BackgroundQueue::Utils.get_hash_entry(env_config, :memcache)
+        if memcache_entry && memcache_entry.kind_of?(String)
+          memcache_entry.split(',').collect { |entry| entry.strip }.select { |entry| !entry.nil? && entry.length > 0 }
+        elsif memcache_entry
+          raise BackgroundQueue::LoadError, "Error loading 'memcache' entry in configuration file #{full_path(path)}: invalid data type (#{memcache_entry.class.name}), expecting String (comma separated)"
+        else
+          raise BackgroundQueue::LoadError, "Missing 'memcache' entry in configuration file #{full_path(path)}"
+        end
+      end
+    end
+  end
+  
+  #Error raised when unable to load configuration
+  class LoadError < Exception
+
+  end
+end

data/lib/background_queue/server_lib/balanced_queue.rb

@@ -0,0 +1,108 @@
+require 'thread'
+module BackgroundQueue::ServerLib
+  
+  class BalancedQueue < PriorityQueue
+    include BackgroundQueue::ServerLib::QueueRegistry
+    
+    attr_reader :server
+    
+    def initialize(server)
+      @task_registry = BackgroundQueue::ServerLib::TaskRegistry.new
+      @condvar = ConditionVariable.new
+      @mutex = Mutex.new
+      @server = server
+      @thread_manager = server.thread_manager
+      super()
+    end
+    
+    def add_task(task)
+      @thread_manager.protect_access {
+        status, existing_task = @task_registry.register(task)
+        if status != :waiting
+          if status == :existing
+            remove_item(existing_task)
+          end
+          add_item(task)
+          @thread_manager.signal_access #wake anything reading from the queue
+        end
+      }
+    end
+    
+    def remove_task(task)
+      @thread_manager.protect_access {
+        remove_item(task)
+      }
+    end
+    
+    def finish_task(task)
+      @thread_manager.protect_access {
+        finish_item(task)
+        existing_task = @task_registry.de_register(task.id)
+        if existing_task
+          add_item(task)
+        end
+      }
+    end
+    
+    def next_task
+      task = nil
+      @thread_manager.control_access {
+        task = next_item
+        if task.nil?
+          @thread_manager.wait_on_access
+        end
+      }
+      task
+    end
+    
+    def self.queue_class
+      BackgroundQueue::ServerLib::Owner
+    end
+    
+    def register_job(job)
+      @server.jobs.register(job)
+    end
+    
+    def synchronous?
+      false
+    end
+    
+    def save_to_file(io)
+      data = []
+      @server.logger.debug("Saving task queue to file")
+      @thread_manager.protect_access {
+        each_item { |owner|
+          owner.each_item { |job|
+            job.each_item { |task|
+              data << task.to_json_object(true)
+            }
+          }
+        }
+      }
+      @server.logger.debug("Writing #{data.length} entries to file")
+      io.write(JSON.fast_generate(data))
+    end
+    
+    def load_from_file(io)
+      @server.logger.debug("Loading task queue from file")
+      tasks = JSON.parse(io.read, :symbolize_names=>true)
+      @server.logger.debug("Adding #{tasks.length} tasks from file")
+      for task_data in tasks
+        task = Task.new(task_data[:owner_id], task_data[:job_id], task_data[:id], task_data[:priority], task_data[:worker], task_data[:params], task_data[:options])
+        add_task(task)
+      end
+    end
+    
+    private
+    
+    def get_queue_id_from_item(item)
+      item.owner_id
+    end
+    
+    def add_item_to_queue(queue, item)
+      queue.add_item(item)
+    end
+    
+
+  end
+end