background_queue 0.2.0

data/lib/background_queue/server_lib/owner.rb

@@ -0,0 +1,54 @@
+module BackgroundQueue::ServerLib
+  
+  class Owner < PriorityQueue
+    include BackgroundQueue::ServerLib::QueueRegistry
+    
+    attr_accessor :id
+    
+    def initialize(id, balanced_queues)
+      @id = id
+      @balanced_queues = balanced_queues
+      super()
+    end
+
+    
+    def ==(other)
+      @id == other.id
+    end
+    
+    def server
+      @balanced_queues.server
+    end
+    
+    def inspect
+      "OwnerQueue:#{self.id} (#{self.object_id})"
+    end
+
+    def self.queue_class
+      BackgroundQueue::ServerLib::Job
+    end
+    
+    def synchronous?
+      false
+    end
+
+    private
+    
+    def create_queue(queue_id)
+      job = super(queue_id)
+      @balanced_queues.register_job(job)
+      job
+    end
+    
+    def get_queue_id_from_item(item)
+      item.job_id
+    end
+    
+    def add_item_to_queue(queue, item)
+      queue.add_item(item)
+    end
+    
+    
+
+  end
+end

data/lib/background_queue/server_lib/priority_queue.rb

@@ -0,0 +1,156 @@
+module BackgroundQueue::ServerLib
+  #internally implemented using a list of queues
+  #this does not do any locking, subclasses should look after any locking
+  class PriorityQueue
+  
+    def initialize
+      @queues = []
+      @items = {}
+      @stalled_items = {}
+      @stalled = false
+    end
+    
+    def pop
+      q = get_next_queue
+      return nil if q.nil?
+      item = q.shift
+      if q.empty?
+        remove_queue(q)
+      end
+      item
+    end
+    
+    def push(item)
+      q = get_queue_for_priority(item.priority, true)
+      q.push(item)
+    end
+    
+    def remove(item, override_priority=nil)
+      override_priority = item.priority if override_priority.nil?
+      q = get_queue_for_priority(override_priority, false)
+      raise "unable to get queue at priority #{override_priority} when removing" if q.nil?
+      q.delete_if { |q_item| q_item.id == item.id }
+      if q.empty?
+        remove_queue(q)
+      end
+    end
+    
+    #the highest priority queue
+    def priority
+      return nil if @queues.empty?
+      get_next_queue.priority
+    end
+    
+    def stalled?
+      @stalled
+    end
+    
+    def stalled=(stall)
+      @stalled = stall
+    end
+    
+    def empty?
+      @queues.empty?
+    end
+    
+    def number_of_priorities
+      @queues.length
+    end
+    
+    def number_if_items_at_priority(priority)
+      q = get_queue_for_priority(priority, false)
+      return 0 if q.nil?
+      q.length
+    end
+    
+    def peek
+      q = get_next_queue
+      return nil if q.nil?
+      q.first
+    end
+    
+    def each_item(&block)
+      for queue in @queues
+        for item in queue
+          block.call(item)
+        end
+      end
+    end
+    
+    private
+    
+    def get_queue_for_priority(priority, create)
+      @queues.each_with_index do |q, idx|
+        return q if q.priority == priority
+        if q.priority > priority #passed it.. insert here...
+          return nil unless create
+          return insert_queue_at_index(priority, idx)
+        end
+      end
+      return insert_queue_at_index(priority, -1)
+    end
+    
+    def insert_queue_at_index(priority, index)
+      q = PriorityArray.new(priority)
+      @queues.insert(index, q)
+      q
+    end
+    
+    def get_next_queue
+      @queues.first
+    end
+    
+    def remove_queue(queue)
+      @queues.delete(queue)
+    end
+    
+    def get_queues
+      @queues
+    end
+  end
+  
+  
+  #this is an array with a priority attribute
+  class PriorityArray < Array
+    
+    attr_accessor :priority
+    def initialize(priority)
+      @priority = priority
+      super(0)
+    end
+    
+    def ==(other)
+      other.priority == self.priority
+    end
+    
+    def inspect
+      "#{self.priority}:#{super}"
+    end
+  end
+end
+
+
+#balanced_queue => owners => jobs => items
+#items can change owner/jobs
+#balanced_queue, and owner are priority queues
+#job is a normal queue
+#as an item is popped from the queue, its priority is recalculated
+
+#bq.next_task gets pops owner with highest priority. Calls owner.next_task(). If no tasks, will wait()
+  #owner.next_task pops next job with highest priority. Calls job.next_task
+    #job.next_task pops next task with highest priority
+  #job is added back to owner with current priority (unless empty?). Task is returned
+#owner is added back to bq with current priority (unless empty?), Task is returned
+  
+#task is run()
+  #mark task as running
+  #call http to call the worker
+  #deregister task if run_version is the same (run_version is incremented each time the same task is reregistered)
+  
+  
+  #task registry keeps track of items so they can be found/re-inserted
+  #the queue keeps track of all items using hashes until item is removed (not popped)
+  #this allows a get(id) method and a pop() method
+  #get will add an instance to the hash if not found
+  
+    

data/lib/background_queue/server_lib/queue_registry.rb

@@ -0,0 +1,123 @@
+module BackgroundQueue::ServerLib
+  module QueueRegistry
+  
+    def add_item(item)
+      in_queue, queue = get_queue(get_queue_id_from_item(item), true)
+      priority_increased, original_priority = track_priority_when_adding_to_queue(queue, item)
+      if !queue.stalled? && (!in_queue || priority_increased)
+        if in_queue #remove from existing priority queue
+          remove(queue, original_priority)
+        end
+        push(queue)
+      end
+    end
+    
+    def next_item
+      item = nil
+      queue = pop
+      unless queue.nil?
+        priority_decreased, original_priority, item = remove_item_from_queue(queue, :next)
+        
+        # some items must run synchronously, so we dont want to add it back until the task is finished.
+        # if the queue it empty we still want to keep it there until the task is finished, incase the running task queues more tasks against the job.
+        if queue.empty? || queue.synchronous?
+          @items.delete(queue.id) 
+          stall_queue(queue)
+        else
+          push(queue)
+        end
+      end
+      item
+    end
+    
+    def remove_item(item)
+      in_queue, queue = get_queue(get_queue_id_from_item(item), false)
+      raise "Unable to remove task #{item.id} at priority #{item.priority} (no queue at that priority)" if queue.nil?
+      priority_decreased, original_priority, item = remove_item_from_queue(queue, item)
+      if queue.empty?
+        remove(queue, original_priority)
+        @items.delete(queue.id)
+      elsif priority_decreased
+        remove(queue, original_priority)
+        push(queue)
+      end
+    end
+    
+    def finish_item(item)
+      #puts "#{self.class.name}:finish item: #{item.inspect}"
+      in_queue, queue = get_queue(get_queue_id_from_item(item), false)
+      raise "Queue #{get_queue_id_from_item(item)} unavailble when finishing item" if queue.nil?
+      queue.finish_item(item)
+      resume_queue(queue)
+    end
+    
+    private
+    
+    def create_queue(queue_id)
+      self.class.queue_class.new(queue_id, self)
+    end
+    
+    def track_priority_when_adding_to_queue(queue, item)
+      original_priority = queue.priority
+      add_item_to_queue(queue, item) #queue.add_item(item)
+       
+      if original_priority.nil? || original_priority > queue.priority
+        return [true, original_priority]
+      end
+      [false, original_priority]
+    end
+    
+    def remove_item_from_queue(queue, target_item)
+      original_priority = queue.priority
+      
+      if target_item == :next
+        item = queue.next_item
+      else
+        item = queue.remove_item(target_item)
+      end
+       
+      if queue.priority.nil? || original_priority < queue.priority
+        return [true, original_priority, item]
+      end
+      [false, original_priority, item]
+    end
+    
+    def get_queue(queue_id, create)
+      queue =  @items[queue_id]
+      queue = @stalled_items[queue_id] if queue.nil?
+      return [true, queue] unless queue.nil?
+      return [false, nil] unless create
+      queue = create_queue(queue_id) #BackgroundQueue::ServerLib::Owner.new(owner_id)
+      @items[queue_id] = queue
+      return [false, queue]
+    end
+    
+    def stall_queue(queue)
+      queue.stalled = true
+      #puts "stalling queue #{queue.inspect}"
+      @stalled_items[queue.id] = queue
+    end
+    
+    def resume_queue(queue)
+      if queue.stalled?
+        @stalled_items.delete(queue.id) 
+        unless queue.empty? 
+          queue.stalled = false
+          push(queue)
+          @items[queue.id] = queue
+          #puts "returned q: #{queue.inspect}"
+        else
+          @items.delete(queue.id)
+          #puts "q empty"
+        end
+      #else
+      #  puts "q not stalled"
+      end
+    end
+    
+    def stalled_items
+      @stalled_items
+    end
+    
+  end
+end

data/lib/background_queue/server_lib/server.rb

@@ -0,0 +1,314 @@
+require 'optparse'
+require 'logger'
+
+module BackgroundQueue::ServerLib
+  class Server
+    
+    attr_accessor :config
+    attr_accessor :thread_manager
+    attr_accessor :task_queue
+    attr_accessor :event_server
+    attr_accessor :workers
+    attr_accessor :jobs
+    attr_accessor :logger
+    
+    def initialize
+      @running = false
+      @stat_mutex = Mutex.new
+      @stats = {
+        :tasks=>0,
+        :run_tasks=>0,
+        :running=>0
+      }
+    end
+    
+    def process_args(argv)
+      argv = argv.clone
+      cmd = argv.shift
+      
+      if cmd.nil?
+        raise BackgroundQueue::ServerLib::InitError, "Usage: server command [options]"
+      end
+      
+      options = {:command=>cmd.nil? ? nil : cmd.downcase.intern}
+      
+      env_to_load = "development"
+      
+      OptionParser.new do |opts|
+        opts.banner = "Usage: server command [options]"
+        case options[:command]
+          when :start, :test
+            opts.on("-c", "--config PATH", "Configuration Path") do |cp|
+              options[:config] = cp
+            end
+          when :stop
+            
+          when nil
+              
+          else
+            raise "Invalid Command: #{cmd}"
+        end
+        opts.on("-l", "--logfile [PATH]", "Logfile Path") do |lf|
+          options[:log_file] = lf
+        end
+        opts.on("-v", "--loglevel [LEVEL]", "Log Level") do |ll|
+          options[:log_level] = ll
+        end
+        opts.on("-p", "--pidfile [PATH]", "Pid file Path (/var/run/background_queue.pid)") do |pf|
+          options[:pid_file] = pf
+        end
+        opts.on("-e", "--environment [RAILS_ENV]", "testing/development/production (development)") do |env|
+          env_to_load = env
+        end
+      end.parse!(argv)
+      
+      ENV['RAILS_ENV']=env_to_load
+      
+      raise BackgroundQueue::ServerLib::InitError, "Missing config argument (-c)" if options[:config].nil? && ([:test, :start].include?(options[:command]) )
+    
+      options
+    end
+    
+    def load_configuration(path)
+      @config = BackgroundQueue::ServerLib::Config.load_file(path)
+      true
+    end
+    
+    def resolve_logging_path(path)
+      File.expand_path(path)
+    end
+    
+    def set_logging_level(log, level)
+      if level.nil? || level.strip.length == 0
+        level = "warn" 
+      else
+        level = level.to_s.downcase
+      end
+      case level
+      when 'debug'
+        log.level = Logger::DEBUG
+      when 'info'
+        log.level = Logger::INFO
+      when 'warn'
+        log.level = Logger::WARN
+      when 'error'
+        log.level = Logger::ERROR
+      when 'fatal'
+        log.level = Logger::FATAL
+      else
+        raise BackgroundQueue::ServerLib::InitError, "Unknown logging level: #{level}"
+      end
+    end
+    
+    def init_logging(path, level)
+      unless path.nil? || path.to_s.strip.length == 0
+        path = resolve_logging_path(path)
+        begin
+          @logger = Logger.new(path, "daily")
+          set_logging_level(@logger, level)
+        rescue Exception=>e
+          raise BackgroundQueue::ServerLib::InitError, "Error initializing log file #{path}: #{e.message}"
+        end
+      end
+      if @logger.nil?
+        #just make a fallback logger...
+        @logger = Logger.new($stderr)
+        set_logging_level(@logger, "fatal")
+      end
+    end
+    
+    def get_pid_path(options)
+      if options[:pid_file]
+        options[:pid_file]
+      else
+        "/var/run/background_queue.pid"
+      end
+    end
+    
+    def get_pid(options)
+      sPid = nil
+      begin
+        sPid = File.open(get_pid_path(options)) { |f|
+          f.read
+        }
+      rescue
+        return nil
+      end
+      return nil if sPid.nil? || sPid.to_i == 0
+      nPid = sPid.to_i
+      begin
+        Process.kill(0, nPid)
+        return nPid
+      rescue
+        return nil
+      end
+    end
+    
+    def check_not_running(options)
+      proc_id = get_pid(options)
+      raise BackgroundQueue::ServerLib::InitError, "Process #{proc_id} already running" unless proc_id.nil?
+      nil
+    end
+    
+    def stop_pid(options)
+      proc_id = get_pid(options)
+      unless proc_id.nil?
+        begin
+          Process.kill(15, proc_id) 
+        rescue 
+          #dont care... the process may have died already?
+        end
+        count = 0
+        while get_pid(options) && count < 10
+          puts "Waiting..."
+          sleep(1)
+        end
+        kill_pid(options) #make sure
+      end
+    end
+    
+    def kill_pid(options)
+      proc_id = get_pid(options)
+      begin
+        Process.kill(9, proc_id) unless proc_id.nil?
+      rescue 
+        #dont care... the process may have died already?
+      end
+    end
+    
+    def write_pid(options)
+      proc_id = Process.pid
+      begin
+        File.open(get_pid_path(options), "w") { |f|
+          f.write(proc_id.to_s)
+        }
+      rescue Exception=>e
+        raise BackgroundQueue::ServerLib::InitError, "Unable to write to pid file #{get_pid_path(options)}: #{e.message}"
+      end
+    end
+    
+    def remove_pid(options)
+      begin
+        File.delete(get_pid_path(options))
+      rescue 
+      end
+    end
+    
+    def trap_signals
+      Signal.trap("TERM") do
+        puts "Terminating..."
+        self.stop()
+      end
+    end
+    
+    
+    def daemonize(options)
+      fork{
+        stdin = open '/dev/null', 'r'
+        stdout = open '/dev/null', 'w'
+        stderr = open '/dev/null', 'w'
+        STDIN.reopen stdin
+        STDOUT.reopen stdout
+        STDERR.reopen stderr
+        fork{
+          write_pid(options) unless options[:skip_pid]
+          run(options)
+        } and exit!
+      }
+    end
+    
+    def start(options)
+      begin
+        load_configuration(options[:config])
+        init_logging(options[:log_file], options[:log_level])
+        check_not_running(options) unless options[:skip_pid]
+        write_pid(options) unless options[:skip_pid] #this will make sure we can write the pid file... the daemon will write it again
+        if options[:command] == :start
+          daemonize(options)
+        elsif options[:command] == :run
+          run(options)
+        else
+          raise BackgroundQueue::ServerLib::InitError, "Unknown Command: #{options[:command]}"
+        end
+      rescue BackgroundQueue::ServerLib::InitError=>ie
+        STDERR.puts ie.message
+      rescue Exception=>e
+        STDERR.puts e.message
+        STDERR.puts e.backtrace.join("\n")
+      end
+    end
+    
+    def running?
+      @running
+    end
+    
+    def run(options)
+      trap_signals
+      @running = true
+      @thread_manager = BackgroundQueue::ServerLib::ThreadManager.new(self, self.config.connections_per_worker)
+      
+      @workers = BackgroundQueue::ServerLib::WorkerBalancer.new(self)
+      @task_queue = BackgroundQueue::ServerLib::BalancedQueue.new(self)
+      
+      @thread_manager.start(BackgroundQueue::ServerLib::WorkerThread)
+      
+      @event_server = BackgroundQueue::ServerLib::EventServer.new(self)
+      
+      @jobs = BackgroundQueue::ServerLib::JobRegistry.new
+      
+      load_tasks(config.task_file)
+      
+      @event_server.start
+    end
+    
+    def stop(timeout_secs=10)
+      @running = false
+      @event_server.stop
+      @thread_manager.wait(timeout_secs)
+      save_tasks(config.task_file)
+    end
+    
+    def change_stat(stat, delta)
+      @stat_mutex.synchronize {
+        @stats[stat] += delta
+      }
+    end
+    
+    def get_stats
+      @stat_mutex.synchronize {
+         @stats.clone
+      }
+    end
+    
+    def load_tasks(path)
+      return if path.nil?
+      if File.exist?(path)
+        begin
+          File.open(path, 'r') { |io| 
+            task_queue.load_from_file(io)
+          }
+        rescue Exception=>e
+          logger.error("Error loading tasks from #{path}: #{e.message}")
+          logger.debug(e.backtrace.join("\n"))
+        end
+      end
+    end
+    
+    def save_tasks(path)
+      return if path.nil?
+      
+      begin
+        File.open(path, 'w') { |io| 
+          task_queue.save_to_file(io)
+        }
+      rescue Exception=>e
+        logger.error("Error saving tasks to #{path}: #{e.message}")
+        logger.debug(e.backtrace.join("\n"))
+      end
+    end
+  end
+  
+  class InitError < Exception
+    
+  end
+end