background_queue 0.2.0

data/lib/background_queue/server_lib/sorted_workers.rb

@@ -0,0 +1,52 @@
+#we want a list of workers where the first in the list is the next worker to use.
+#the next worker to use is the worker with the least number of running connections
+
+class BackgroundQueue::ServerLib::SortedWorkers
+
+  attr_reader :worker_list
+  
+  def initialize
+    @worker_list = []
+  end
+  
+  #add the worker back in the correct position
+  def add_worker(worker)
+    idx = 0
+    while idx < @worker_list.length && @worker_list[idx].connections < worker.connections
+      idx += 1
+    end
+    if idx == 0
+      @worker_list.unshift(worker)
+    else
+      @worker_list.insert(idx, worker)
+    end
+  end
+  
+  def remove_worker(worker)
+    @worker_list.delete(worker)
+  end
+  
+  
+  def adjust_worker(worker)
+    idx = @worker_list.index(worker)
+    raise "Worker not found (#{worker.inspect} not in #{@worker_list.inspect})" if idx.nil?
+    swap_idx = idx - 1
+    while swap_idx >= 0 && @worker_list[swap_idx].connections > worker.connections
+      swap_idx -= 1
+    end
+    swap_idx += 1
+    if swap_idx == idx #we didnt move forward, try backwards
+      swap_idx = idx + 1
+      while swap_idx < @worker_list.length && @worker_list[swap_idx].connections < worker.connections
+        swap_idx += 1
+      end
+      swap_idx -= 1
+    end
+    if swap_idx != idx
+      tmp = @worker_list[swap_idx]
+      @worker_list[swap_idx] = @worker_list[idx]
+      @worker_list[idx] = tmp
+    end
+  end
+  
+end

data/lib/background_queue/server_lib/task.rb

@@ -0,0 +1,79 @@
+module BackgroundQueue::ServerLib
+  class Task
+    
+    attr_accessor :id
+    attr_accessor :priority
+    
+    attr_accessor :owner_id
+    attr_accessor :job_id
+    
+    attr_accessor :worker
+    
+    attr_accessor :running
+    
+    attr_accessor :options
+  
+    def initialize(owner_id, job_id, id, priority, worker, params, options)
+      @owner_id = owner_id
+      @job_id = job_id
+      @id = id
+      @priority = priority
+      @worker = worker
+      @running = false
+      @options = options
+      @params = params
+    end
+    
+    def to_json(dummy=true)
+      to_json_object(false).to_json
+    end
+    
+    def to_json_object(full)
+      jo = {:owner_id=>@owner_id, :job_id=>@job_id, :id=>@id, :priority=>@priority, :worker=>@worker, :params=>@params }
+      if full
+        jo[:options] = @options
+      end
+      jo
+    end
+    
+    def running?
+      @running
+    end
+    
+    def domain
+      @options[:domain]
+    end
+    
+    def set_job(job)
+      @job = job
+    end
+    
+    def is_excluded_from_count?
+      @options[:exclude] == true
+    end
+    
+    def synchronous?
+      @options[:synchronous] == true
+    end
+    
+    def weighted?
+      @options[:weight] && @options[:weight] > 0
+    end
+    
+    def weighted_percent
+      @options[:weight]
+    end
+    
+    def initial_progress_caption
+      @options[:initial_progress_caption]
+    end
+    
+    def set_worker_status(status)
+      raise "Task without job set" if @job.nil?
+      status[:task_id] = self.id
+      status[:exclude] = self.is_excluded_from_count?
+      status[:weight] = self.weighted_percent if self.weighted?
+      @job.set_worker_status(status)
+    end
+  end
+end

data/lib/background_queue/server_lib/task_registry.rb

@@ -0,0 +1,51 @@
+module BackgroundQueue::ServerLib
+  
+  #keep track if tasks already queued and running so if the same task comes in, we know to remove it.
+  class TaskRegistry
+  
+    def initialize
+      @waiting_tasks = {}
+      @tasks = {}
+    end
+    
+    def register(task)
+      existing_task = @tasks[task.id]
+      if existing_task.nil? 
+        @tasks[task.id] = task
+        [:new, nil]
+      elsif existing_task.running?
+        register_waiting_task(task)
+        [:waiting, nil]
+      else
+        @tasks[task.id] = task
+        [:existing, existing_task]
+      end
+    end
+    
+    def de_register(task_id)
+      @tasks.delete(task_id)
+      waiting = get_waiting_task(task_id)
+      if waiting
+        @tasks[task_id] = waiting
+      end
+      waiting
+    end
+    
+    def register_waiting_task(task)
+      @waiting_tasks[task.id] = task
+    end
+    
+    def get_waiting_task(task_id)
+      @waiting_tasks.delete(task_id)
+    end
+    
+    def waiting_tasks
+      @waiting_tasks
+    end
+    
+    def tasks
+      @tasks
+    end
+  end
+
+end

data/lib/background_queue/server_lib/thread_manager.rb

@@ -0,0 +1,121 @@
+#make sure threads are schedules and the max number of threads is controlled
+class BackgroundQueue::ServerLib::ThreadManager
+
+  attr_accessor :max_threads
+  attr_reader :running_threads
+  
+  def initialize(server, max_threads)
+    @server = server
+    @max_threads = max_threads
+    @running_threads = 0
+    @mutex = Mutex.new
+    @condvar = ConditionVariable.new
+    @threads = []
+  end
+  
+  def protect_access(&block)
+    @mutex.synchronize {
+      block.call
+    }
+  end
+
+  def control_access(&block)
+    @mutex.synchronize {
+      if @running_threads >= @max_threads && @server.running?
+        @running_threads -= 1
+        @condvar.wait(@mutex)
+        @running_threads += 1
+      end
+      block.call
+    }
+  end
+  
+  #signal any waiting threads
+  #this should only be called from with a protect_access/control_access block
+  #will do nothing if there are already too many threads running
+  def signal_access
+    @condvar.signal unless @running_threads >= @max_threads
+  end
+  
+  #wait for the condition
+  #must be called from within protect_access/control_access block
+  def wait_on_access
+    if @server.running?
+      @running_threads -= 1
+      #puts "waiting"
+      @condvar.wait(@mutex)
+      #puts "woken"
+      @running_threads += 1
+    end
+  end
+  
+  def change_concurrency(max_threads)
+    @mutex.synchronize {
+      if max_threads > @max_threads
+        for i in @max_threads...max_threads
+          @condvar.signal
+        end
+      end
+      @max_threads = max_threads
+    }
+  end
+  
+  
+  def start(clazz)
+    @mutex.synchronize {
+      for i in 0...@max_threads
+        runner = clazz.new(@server)
+        @running_threads += 1
+        #puts "started thread, running=#{@running_threads}"
+        @threads << Thread.new(runner) { |runner|
+          begin
+            runner.run
+          rescue Exception=>e
+            @server.logger.error("Error in thread: #{e.message}")
+            @server.logger.debug(e.backtrace.join("\n"))
+          end
+          @mutex.synchronize {
+            @running_threads -= 1
+            #puts "finished thread, running=#{@running_threads}"
+          }
+        }
+      end
+    }
+  end
+  
+  def wait(timeout_limit = 100)
+    #for thread in @threads
+      @mutex.synchronize {
+        @condvar.broadcast
+      }
+    #end
+    #while @running_threads > 0
+    #  @mutex.synchronize {
+    #    @condvar.signal
+    #  }
+    #  sleep(0.01)
+    #end
+    begin
+      Timeout::timeout(timeout_limit) {
+        for thread in @threads
+          thread.join
+        end
+      }
+    rescue Timeout::Error => te
+      for thread in @threads
+        begin
+          if thread.alive?
+            thread.raise BackgroundQueue::ServerLib::ThreadManager::ForcedStop.new("Timeout when forcing threads to stop")
+          end
+        rescue Exception=>e
+          #ignore
+        end
+      end
+    end
+  end
+  
+  #Error raised when unable to load configuration
+  class ForcedStop < Exception
+
+  end
+end

data/lib/background_queue/server_lib/worker.rb

@@ -0,0 +1,18 @@
+#this is a worker which keeps track of how many connections are getting used by the worker.
+class BackgroundQueue::ServerLib::Worker
+
+  attr_accessor :uri
+  attr_accessor :connections
+  attr_accessor :offline
+  
+  def initialize(uri)
+    @uri = uri
+    @connections = 0
+    @offline = false
+  end
+  
+  def offline?
+    @offline
+  end
+
+end

data/lib/background_queue/server_lib/worker_balancer.rb

@@ -0,0 +1,97 @@
+module BackgroundQueue::ServerLib
+  #make sure each worker gets its fair share of tasks
+  #track the number of active connections to use as the balancing metric
+  class WorkerBalancer
+    
+    attr_reader :available_workers
+    attr_reader :offline_workers
+    
+    def initialize(server)
+      @server = server
+      @mutex = Mutex.new
+      @offline_workers = []
+      @available_workers = SortedWorkers.new
+      for worker_config in server.config.workers.reverse
+        worker = Worker.new(worker_config.uri)
+        @available_workers.add_worker(worker)
+      end
+    end
+    
+    #poll the workers that are marked as offline, and mark them online if the polling succeeded
+    def check_offline
+      
+      workers_to_check = @mutex.synchronize { @offline_workers.clone }
+
+      for worker in workers_to_check
+        client = BackgroundQueue::ServerLib::WorkerClient.new(@server)
+        if client.send_request(worker, build_poll_task, @server.config.secret)
+          register_online(worker)
+        end
+      end
+    end
+    
+    #get the worker with the least number of connections using it
+    def get_next_worker
+      @mutex.synchronize { 
+        worker = @available_workers.worker_list.first 
+        unless worker.nil?
+          register_start(worker)
+        end
+        worker
+      }
+    end
+    
+    def finish_using_worker(worker, online)
+      @mutex.synchronize { 
+        if online
+          register_finish(worker)
+        else
+          register_offline(worker)
+        end
+      }
+    end
+    
+    private
+    
+    def register_start(worker)
+      worker.connections += 1
+      @available_workers.adjust_worker(worker)
+    end
+    
+    def register_finish(worker)
+      worker.connections -= 1
+      @available_workers.adjust_worker(worker)
+    end
+    
+    def register_offline(worker)
+      worker.connections -= 1
+      unless worker.offline?
+        worker.offline = true
+        @available_workers.remove_worker(worker)
+        @offline_workers << worker
+      end
+    end
+    
+    def register_online(worker)
+      if worker.offline?
+        worker.offline = false
+        @offline_workers.delete(worker)
+        @available_workers.add_worker(worker)
+      end
+    end
+    
+    def build_poll_task
+      if @poll_task.nil?
+        @poll_task = BackgroundQueue::ServerLib::Task.new(:owner_id, :job_id, :id, 1, :poll_worker, {}, @server.config.system_task_options)
+        @poll_task.set_job(BackgroundQueue::ServerLib::NullJob.new)
+      end
+      @poll_task
+    end
+  end
+  
+  class NullJob
+    def set_worker_status(status)
+      #do nothing
+    end
+  end
+end

data/lib/background_queue/server_lib/worker_client.rb

@@ -0,0 +1,94 @@
+require 'net/http'
+require 'json'
+
+module BackgroundQueue::ServerLib
+  #The client to a worker.
+  #Use http to connect to the worker, send the command, and process the streamed response of json encoded status updates.
+  class WorkerClient
+    def initialize(server)
+      @server = server
+    end
+    
+    #send a request to the specified worker, passing the task and authenticating using the secret
+    def send_request(worker, task, secret)
+      @current_task = task
+      req = build_request(worker.uri, task, secret)
+      begin
+        Net::HTTP.start(worker.uri.host, worker.uri.port)  do |server|
+          server.request(req) do |response|
+            read_response(worker, response, task)
+          end
+        end
+        true
+      rescue Exception=>e
+        @server.logger.error("Error sending request #{task.id} to worker: #{e.message}")
+        @server.logger.debug(e.backtrace.join("\n"))
+        return false
+      end
+    end
+    
+    private
+    
+    def build_request(uri, task, secret)
+      req = Net::HTTP::Post.new(uri.path)
+      req.set_form_data({:task=>task.to_json, :auth=>secret, :server_port=>@server.config.address.port})
+      req["host"] = task.domain
+      req
+    end
+    
+    
+    
+    def read_response(worker, http_response, task)
+      if http_response.code == "200"
+        http_response.read_body do |chunk|
+          process_chunk(chunk, task)
+        end
+        #the last chunk did not end in a newline... process it
+        unless @prev_chunk.nil?
+          process_line(@prev_chunk.strip, task)
+          @prev_chunk = nil
+        end
+      else
+        raise "Invalid response code (#{http_response.code}) when calling #{worker.uri.to_s}"
+      end
+    end
+    
+    def process_chunk(chunk, task)
+      #puts "process_chunk: #{chunk}"
+      chunk.each_line do |line|
+        unless @prev_chunk.nil?
+          line = @prev_chunk + line
+          @prev_chunk = nil
+        end
+        if line[-1,1] == "\n" #it ends in a newline so its a complete line
+          process_line(line.strip, task)
+        else
+          @prev_chunk = line
+        end
+      end
+    end
+    
+    def process_line(line, task)
+      hash_data = nil
+      begin
+        hash_data = JSON.load(line)
+        if hash_data.kind_of?(Hash)
+          set_worker_status(hash_data, task)
+        else
+          raise "Invalid Status Line (wrong datatype: #{hash_data.class.name}"
+        end
+        true
+      rescue Exception=>e
+        @server.logger.error("Error processing status line of task #{task.id}: #{e.message}")
+        @server.logger.debug(e.backtrace.join("\n"))
+        false
+      end
+    end
+    
+    def set_worker_status(status, task)
+      status_map = BackgroundQueue::Utils::AnyKeyHash.new(status)
+      task.set_worker_status(status_map)
+    end
+    
+  end
+end