background_queue 0.2.0

data/lib/background_queue/server_lib/event_server.rb

@@ -0,0 +1,35 @@
+require 'eventmachine'
+require 'rubygems'
+require 'rufus/scheduler'
+
+module BackgroundQueue::ServerLib
+  class EventServer
+    
+    attr_reader :running
+    
+    def initialize(server)
+      @server = server
+      @running = false
+    end
+    
+    def start
+      EventMachine.run do
+        EventMachine::start_server(@server.config.address.host, @server.config.address.port, BackgroundQueue::ServerLib::EventConnection) do |conn|
+          conn.server = @server
+        end
+
+        @scheduler = Rufus::Scheduler::EmScheduler.new
+        @scheduler.start
+        for job in @server.config.jobs
+          job.schedule(@scheduler, @server)
+        end
+        @running = true
+      end
+      @running = false
+    end
+    
+    def stop
+      EventMachine::stop if EventMachine::reactor_running?
+    end
+  end
+end

data/lib/background_queue/server_lib/job.rb

@@ -0,0 +1,252 @@
+module BackgroundQueue::ServerLib
+  class Job < PriorityQueue
+  
+    attr_accessor :id
+    attr_reader :running_status
+    attr_reader :running_ordered_status
+    attr_reader :total_tasks
+    attr_reader :total_counted_tasks
+    attr_reader :completed_tasks
+    attr_reader :completed_counted_tasks
+    attr_reader :running_percent
+    attr_reader :running_percent_counted
+    attr_reader :current_running_status
+    
+    attr_reader :total_weighted_tasks
+    attr_reader :total_weighted_percent
+    attr_reader :completed_weighted_percent
+    attr_reader :completed_weighted_tasks
+    attr_reader :running_percent_weighted
+    
+    #attr_reader :current_running_excluded_status
+    
+    def initialize(id, owner)
+      @id = id
+      @owner = owner
+      @stalled = false
+      @total_tasks = 0
+      @total_counted_tasks = 0
+      @completed_tasks = 0
+      @completed_counted_tasks = 0
+      @running_status = {}
+      @running_ordered_status = []
+      @running_percent = 0
+      @current_running_status = nil
+      @current_progress = {:percent=>0.0, :caption=>""}
+      @current_caption = ""
+      @synchronous_count = 0
+      @total_weighted_tasks = 0
+      @total_weighted_percent = 0.0
+      @completed_weighted_percent = 0.0
+      @completed_weighted_tasks = 0
+      @running_percent_weighted = 0.0
+      @status_meta = {}
+      @mutex = Mutex.new
+      #@current_running_excluded_status = nil
+      super()
+    end
+
+    def ==(other)
+      @id == other.id
+    end
+    
+    def inspect
+      "#{self.id}:#{@queues.inspect}"
+    end
+    
+    def server
+      @owner.server
+    end
+
+    def add_item(task)
+      task.set_job(self)
+      @total_tasks += 1
+      unless task.is_excluded_from_count?
+        @total_counted_tasks += 1
+      end
+      if task.weighted?
+        @total_weighted_tasks += 1
+        @total_weighted_percent += task.weighted_percent
+      end
+      @synchronous_count+=1 if task.synchronous?
+      unless task.initial_progress_caption.nil? || task.initial_progress_caption.length == 0 || @current_progress[:percent] > 0
+        @current_progress[:caption] = task.initial_progress_caption
+      end
+      push(task)
+    end
+    
+    def next_item
+      pop
+    end
+    
+    def finish_item(item)
+      @synchronous_count-=1 if item.synchronous?
+    end
+    
+    def synchronous?
+      @synchronous_count > 0
+    end
+    
+    def set_worker_status(status)
+      if status[:meta]
+        update_status_meta(status[:meta])
+      else
+        running_status = get_running_status(status)
+        if status[:percent] >= 100
+          update_finished_status(status)
+        else
+          update_running_status(running_status, status)
+        end
+      end
+    end
+    
+    def get_running_status(status)
+      rstatus = @running_status[status[:task_id]]
+      rstatus = register_running_status(status) if rstatus.nil?
+      rstatus
+    end
+    
+    def register_running_status(status)
+      rstatus = {:task_id=>status[:task_id], :caption=>status[:caption], :percent=>0, :exclude=>status[:exclude], :weight=>status[:weight] }
+      @running_status[status[:task_id]] = rstatus
+      #if status[:exclude]
+      #  @current_running_excluded_status = rstatus
+      #else
+        @running_ordered_status << rstatus 
+      #end
+      rstatus
+    end
+    
+    def deregister_running_status(task_id)
+      rstatus = @running_status.delete(task_id)
+      unless rstatus.nil?
+        @running_ordered_status.delete(rstatus) 
+      end
+      rstatus
+    end
+    
+    def update_running_status(running_status, status)
+      running_status[:percent] = status[:percent]
+      running_status[:caption] = status[:caption]
+      update_running_percent
+    end
+    
+    
+    def update_status_meta(meta)
+      [:notice, :warning, :error].each { |key|
+        if meta[key]
+          @status_meta[key] = [] if @status_meta[key].nil?
+          @status_meta[key] << meta[key]
+        end
+      }
+      if meta[:meta]
+        @status_meta[:meta] = {} if @status_meta[:meta].nil?
+        @status_meta[:meta] = @status_meta[:meta].update(meta[:meta])
+      end
+      update_current_progress
+    end
+    
+    def update_finished_status(status)
+      rstatus = deregister_running_status(status[:task_id])
+      unless rstatus.nil?
+        @completed_tasks += 1
+        @completed_counted_tasks += 1 unless rstatus[:exclude]
+        unless rstatus[:weight].nil?
+          @completed_weighted_percent += rstatus[:weight] 
+          @completed_weighted_tasks += 1
+        end
+        if self.current_running_status.nil? || @current_running_status == rstatus
+          #sometimes the status is finished straight away...
+          update_current_caption(status)
+        end
+        update_running_percent()
+      end
+    end
+    
+    def update_running_percent
+      total_percent = 0.0
+      total_task_percent = 0.0
+      total_percent_counted = 0.0
+      total_weighted_percent = 0.0
+      for status in @running_ordered_status
+        if status[:weight] && status[:weight] > 0
+          total_weighted_percent += (status[:percent] * status[:weight] / 100.0)
+        else
+          total_percent_counted += status[:percent] unless status[:exclude]
+          total_percent += status[:percent]
+        end
+        total_task_percent += status[:percent]
+      end
+      set_running_percent(total_percent_counted.to_f / 100.0, total_percent.to_f / 100.0, total_task_percent / 100.0, total_weighted_percent.to_f / 100.0)
+      self.update_current_progress
+    end
+    
+    def set_running_percent(pcent_counted, pcent, running_task_pcent, weighted_percent)
+      @running_percent_counted = pcent_counted
+      @running_percent = pcent
+      @running_percent_weighted = weighted_percent
+      idx = running_task_pcent.to_i
+      if @running_ordered_status.length <= idx
+        @current_running_status = @running_ordered_status.last
+      else
+        @current_running_status = @running_ordered_status[idx]
+      end
+    end
+    
+    def get_current_progress_percent
+      unweighted_percent = (100.0 - self.total_weighted_percent) / 100.0
+
+      total_unweighted_tasks = self.total_tasks - self.total_weighted_tasks
+      completed_unweighted_tasks = self.completed_tasks - self.completed_weighted_tasks
+      total_finished_percent = total_unweighted_tasks == 0 ? 0 : (completed_unweighted_tasks.to_f / total_unweighted_tasks.to_f) * 100.0 
+      running_fraction = total_unweighted_tasks == 0 ?  1.0 : (1.0 / total_unweighted_tasks.to_f)
+      
+      
+      total_running_percent = self.running_percent.to_f * running_fraction * 100.0 
+      
+      total_unweighted_percent = (total_finished_percent + total_running_percent) * unweighted_percent
+      
+      total_percent = total_unweighted_percent.to_f + (running_percent_weighted * 100.0) + completed_weighted_percent
+      
+      total_percent
+    end
+    
+    def get_current_progress_caption
+      if self.current_running_status
+        update_current_caption(self.current_running_status)
+      end
+      @current_caption
+    end
+    
+    def update_current_caption(status)
+      caption = status[:caption]
+      caption = "" if caption.nil?
+      if total_counted_tasks > 1 && status[:exclude] != true
+        caption = "#{caption} (#{self.get_current_counted_tasks}/#{self.total_counted_tasks})"
+      end
+      @current_caption = caption
+    end
+    
+    def get_current_counted_tasks
+      cnt = self.completed_counted_tasks + self.running_percent_counted.to_i 
+      if cnt < self.total_counted_tasks
+        cnt += 1
+      end
+      cnt
+    end
+    
+    def update_current_progress
+      @current_progress = {
+        :percent=>get_current_progress_percent,
+        :caption=>get_current_progress_caption
+      }.update(@status_meta)
+      #puts "set status to #{@current_progress.inspect}"
+    end
+
+    def get_current_progress
+      @current_progress
+    end
+    
+   
+  end
+end

data/lib/background_queue/server_lib/job_registry.rb

@@ -0,0 +1,30 @@
+module BackgroundQueue::ServerLib
+  #keep track of jobs, even after they have finished.
+  #this allows us to query the status of the job after its finished.
+  #It is used to get the status of the job
+  class JobRegistry
+    def initialize(size = 1000)
+      @lru = Cache::LRU.new(:max_elements=>size)
+      @mutex = Mutex.new
+      
+    end
+  
+    def register(job)
+      @mutex.synchronize {
+        @lru[job.id] = job
+      }
+    end
+    
+    def get_job(id)
+      @mutex.synchronize {
+        @lru[id]
+      }
+    end
+    
+    private
+    
+    def lru
+      @lru
+    end
+  end
+end

data/lib/background_queue/server_lib/lru.rb

@@ -0,0 +1,193 @@
+# Copyright (c) 2007 Michael Bryzek
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+module Cache
+
+  # Defines a local LRU cache. All operations are maintained in
+  # constant size regardless of the size of the Cache. When creating
+  # the cache, you specify the max size of the cache.
+  #
+  # Example:
+  #   cache = Cache::LRU.new(:max_elements => 5)
+  #    cache.put(:a, 1)
+  #    cache[:a] = 2
+  #    cache.get(:b) { 1 }
+  #    cache[:b]
+  class LRU
+
+    include Enumerable
+
+    attr_reader :size
+    attr_reader :keys
+    # opts:
+    #  - max_elements - maximum number of elements to keep in the
+    #    cache at any time. The default is 100 elements
+    def initialize(opts = {})
+      opts = { :max_elements => 100 }.merge(opts)
+      @max_elements = opts.delete(:max_elements)
+      raise "Invalid options: #{opts.keys.join(' ')}" if opts.keys.size > 0
+      @keys = LinkedList.new
+      @map = {}
+      @size = 0
+    end
+
+    def clear!
+      initialize( :max_elements => @max_elements )
+    end
+    
+    # Iterates through all of the key/value pairs added to the cache,
+    # in random order. Accepts a block that is yielded to with the key
+    # and value for each entry in the cache.
+    def each
+      @map.each do |k, el| 
+        yield k, el.value
+      end
+    end
+
+    def [](key)
+      get(key)
+    end
+    
+    # Fetches the value of the element with the given key. If this key
+    # does not exist in the cache, you can provide an optional code
+    # block that we'll yield to to repopulate the value
+    def get(key)
+      if el = @map[key]
+        @keys.move_to_head(el)
+        return el.value
+      elsif block_given?
+        return put(key, yield)
+      end
+      return nil
+    end
+
+    def []=(key, value)
+      put(key, value)
+    end
+    
+    def put(key, value)
+      el = @map[key]
+      if el
+        el.value = value
+        @keys.move_to_head(el)
+      else
+        el = @keys.add(key, value)
+        @size += 1
+      end
+      @map[key] = el
+
+      if @size > @max_elements
+        delete_element(@keys.last) 
+        @size -= 1
+      end
+      value
+    end
+
+    def delete(key) 
+      if el = @map[key]
+        delete_element(el)
+        @size -= 1
+      else
+        nil
+      end
+    end
+
+    private
+    def delete_element(el)
+      @keys.remove_element(el)
+      @map.delete(el.key)
+      el.value
+    end
+
+  end
+
+  class LinkedList
+
+    attr_reader :last
+    def initialize
+      @head = @last = nil
+    end
+
+    def add(key, value)
+      add_element(Element.new(key, value, @head))
+    end
+
+    def add_element(el)
+      @head.previous_element = el if @head
+
+      el.next_element = @head
+      el.previous_element = nil
+      @head = el
+      @last = el unless @last
+      el
+    end
+
+    def remove_element(el)
+      el.previous_element.next_element = el.next_element if el.previous_element
+      el.next_element.previous_element = el.previous_element if el.next_element
+
+      @last = el.previous_element if el == @last
+      @head = el.next_element if el == @head
+    end
+
+    def move_to_head(el)
+      remove_element(el)
+      add_element(el)
+    end
+
+    # Returns a nicely formatted stirng of all elements in the linked
+    # list. First element is most recently used, last element is least
+    # recently used.
+    def pp
+      s = ''
+      el = @head
+      while el
+        s << ', ' if s.size > 0
+        s <<  el.to_s
+        el = el.next_element
+      end
+      s
+    end
+
+    class Element
+
+      attr_accessor :key, :value, :previous_element, :next_element
+      def initialize(key, value, next_element)
+        @key = key
+        @value = value
+        @next_element = next_element
+        @previous_element = nil
+      end
+
+      def inspect
+        to_s
+      end
+
+      def to_s
+        p = @previous_element ? @previous_element.key : 'nil'
+        n = @next_element ? @next_element.key : 'nil'
+        "[#{@key}: #{@value.inspect}, previous: #{p}, next: #{n}]"
+      end
+
+    end
+
+  end
+
+end