background_queue 0.3.0 → 0.6.0

data/TODO

@@ -1,2 +1,15 @@
 Server
     Raise priority of tasks that have been waiting a long time
+    
+Add shortcut to init and finish tasks
+Add ability to access "summary" data of the tasks in the finish task.
+  summary data operations:
+    append_summary_data(type, data)
+    set_summary_data(type, key, data)
+    increment_summary_data(type, amount)
+    decrement_sumary_data(type, amount)
+    reset_summary_data(type)
+Add cache for tasks in a job to share? use memcache?
+
+Progress Manager
+

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.6.0

data/background_queue.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "background_queue"
-  s.version = "0.3.0"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["MarkPent"]
-  s.date = "2012-10-17"
+  s.date = "2012-11-13"
   s.description = "Organise background tasks so they will not overload the machine(s) running the tasks, while still giving a fair, balanced allocation of running time to members in the queue"
   s.email = "mark.pent@gmail.com"
   s.executables = ["bg_queue"]
@@ -63,6 +63,8 @@ Gem::Specification.new do |s|
     "lib/background_queue/worker/calling.rb",
     "lib/background_queue/worker/config.rb",
     "lib/background_queue/worker/environment.rb",
+    "lib/background_queue/worker/logger.rb",
+    "lib/background_queue/worker/progress.rb",
     "lib/background_queue/worker/worker_loader.rb",
     "lib/background_queue_server.rb",
     "lib/background_queue_worker.rb",
@@ -77,6 +79,7 @@ Gem::Specification.new do |s|
     "spec/background_queue/server_lib/error_task_list_spec.rb",
     "spec/background_queue/server_lib/event_connection_spec.rb",
     "spec/background_queue/server_lib/event_server_spec.rb",
+    "spec/background_queue/server_lib/integration/error_handling_spec.rb",
     "spec/background_queue/server_lib/integration/full_test_spec.rb",
     "spec/background_queue/server_lib/integration/queue_integration_spec.rb",
     "spec/background_queue/server_lib/integration/serialize_spec.rb",
@@ -96,12 +99,15 @@ Gem::Specification.new do |s|
     "spec/background_queue/worker/base_spec.rb",
     "spec/background_queue/worker/calling_spec.rb",
     "spec/background_queue/worker/environment_spec.rb",
+    "spec/background_queue/worker/logger_spec.rb",
+    "spec/background_queue/worker/progress_spec.rb",
     "spec/background_queue/worker/worker_loader_spec.rb",
     "spec/resources/config-client.yml",
     "spec/resources/config-serialize.yml",
     "spec/resources/config.yml",
     "spec/resources/example_worker.rb",
     "spec/resources/example_worker_with_error.rb",
+    "spec/resources/summary_worker.rb",
     "spec/resources/test_worker.rb",
     "spec/shared/queue_registry_shared.rb",
     "spec/spec_helper.rb",

data/lib/background_queue/client.rb

@@ -10,7 +10,25 @@ module BackgroundQueue
       @config = BackgroundQueue::ClientLib::Config.load_file(path)
     end
     
-    #add a task to the background
+    #Add a task to the background
+    #
+    # @param [Symbol] worker name of the worker, ie :some_background_worker
+    # @param [Symbol, String, Number] owner_id something that identified the owner of the task. This will make sure resources are divided between owners equally.
+    # @param [Symbol, String, Number] job_id something to idetify the job. Tracking occurs per job.
+    # @param [Symbol, String, Number] task_id a globally unique id for the task. If the task_id exists elsewhere, it will be removed and added to the owner/job queue specified.
+    # @param [Integer] priority priority for 1 (highest) to 5 (lowest). Used to determine order of jobs.
+    # @param [Hash] task_parameters a hash of parameters passed to the task
+    # @param [Hash] options a hash of options that effect how the task is executed.
+    # @option options [String] :domain the domain to set in the host header when calling the worker.
+    # @option options [Boolean] :exclude if true, will not be included in (x/y) counter of progress caption
+    # @option options [Boolean] :synchronous if true, the task is synchronous, and no other tasks in the job will run until it is finished
+    # @option options [Number] :weight the weight of the task. Usually its weight is the same as other tasks in job
+    # @option options [String] :initial_progress_caption the progress caption to display until the job has started reporting progress
+    # @option options [Boolean] :send_summary if true, the task will receive the summary data
+    # @option options [Symbol] :step the step to run, `:start`, `:run` (Default) or `:finish` 
+    #
+    # @return [BackgroundQueue::ClientLib::JobHandle] A handle to the job which can be used in get_status
+    
     def add_task(worker, owner_id, job_id, task_id, priority, task_parameters={}, options={}, server=nil )
       job_id, task_id = generate_ids(worker, owner_id, job_id, task_id)
       result, server = send_command(BackgroundQueue::ClientLib::Command.add_task_command(worker, owner_id, job_id, task_id, priority, task_parameters, options ), server)
@@ -18,9 +36,19 @@ module BackgroundQueue
       BackgroundQueue::ClientLib::JobHandle.new(owner_id, job_id, server)
     end
     
-    #add multiple tasks to the background, all with the same worker/owner/job
+    #Add multiple tasks to the background, all with the same worker/owner/job
+    #
+    # @param [Symbol] worker name of the worker, ie :some_background_worker
+    # @param [Symbol, String, Number] owner_id something that identified the owner of the task. This will make sure resources are divided between owners equally.
+    # @param [Symbol, String, Number] job_id something to idetify the job. Tracking occurs per job.
+    # @param [Array<Array<String, Hash, Hash>>] tasks an array of arrays in the format [task_id, optional task_params (Hash), optional task_options (Hash)]
+    # @param [Integer] priority priority for 1 (highest) to 5 (lowest). Used to determine order of jobs.
+    # @param [Hash] shared_parameters a hash of parameters passed to the tasks. This is merged with the task_params specified in the tasks param.
+    # @param [Hash] options a hash of options that effect how the tasks are executed. This is merged with the task_options specified in the tasks param. Refer to {#add_task} for options.
     def add_tasks(worker, owner_id, job_id, tasks, priority, shared_parameters={}, options={}, server=nil )
-      send_command(BackgroundQueue::ClientLib::Command.add_tasks_command(worker, owner_id, job_id, tasks, priority, shared_parameters, options ), server)
+      result, server = send_command(BackgroundQueue::ClientLib::Command.add_tasks_command(worker, owner_id, job_id, tasks, priority, shared_parameters, options ), server)
+      #the server currently either returns :ok or an exception would have been thrown
+      BackgroundQueue::ClientLib::JobHandle.new(owner_id, job_id, server)
     end
     
     def get_status(job_handle, options={})

data/lib/background_queue/client_lib/connection.rb

@@ -17,6 +17,13 @@ module BackgroundQueue::ClientLib
       send_with_header(command.to_buf)
       response = receive_with_header
       BackgroundQueue::Command.from_buf(response)
+    ensure
+      begin
+        @socket.close unless @socket.nil?
+      rescue Exception=>e
+        #dont care...
+      end
+      @socket = nil
     end
     
     private
@@ -25,6 +32,8 @@ module BackgroundQueue::ClientLib
       begin
         Timeout::timeout(3) {
           @socket = TCPSocket.open(@server.host, @server.port)
+          linger = [1,0].pack('ii')
+          @socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, linger) #force close so if called many times it wont clog the available ports
           true
         }
       rescue Timeout::Error

data/lib/background_queue/client_lib/job_handle.rb

@@ -1,6 +1,7 @@
+require 'digest/md5'
 module BackgroundQueue::ClientLib
   #returned from add_task to describe what the job/server was added.
-  #this is becuase you can call add_task without a job_id, and not know what server was used.
+  #this is because you can call add_task without a job_id, and not know what server was used.
   #this is passed to get_status
   class JobHandle
     
@@ -13,6 +14,38 @@ module BackgroundQueue::ClientLib
       @job_id = job_id
       @server = server
     end
+    
+    #register this job and return the registered key
+    def register(session_id)
+      md5 = Digest::MD5::new
+      now = Time::now
+      md5.update(now.to_s)
+      md5.update(String(now.usec))
+      md5.update(String(rand(0)))
+      md5.update(String($$))
+      md5.update('foobar')
+      md5.update(owner_id.to_s)
+      md5.update(job_id.to_s)
+      md5.update(server.to_s)
+      key = md5.hexdigest
+          
+      Cache.put("#{session_id}_#{key}", self )
+      
+      reverse_key = [owner_id, job_id, server].join("_")
+      Cache.put("#{session_id}_#{reverse_key}", key )
+    end
+    
+    #look up a job from the key returned from register
+    def self.get_registered_job(session_id, key)
+      Cache.get("#{session_id}_#{key}")
+    end
+    
+    #find the key for this job if its already registeed
+    def get_registration_key(session_id)
+      reverse_key = [owner_id, job_id, server].join("_")
+      Cache.get("#{session_id}_#{reverse_key}")
+    end
+    
   
   end
 

data/lib/background_queue/config.rb

@@ -67,8 +67,6 @@ module BackgroundQueue
         end
       end
       
-      
-      
       def convert_yaml_to_hash(string, path)
         begin
           result = YAML::load(string)
@@ -79,16 +77,8 @@ module BackgroundQueue
         end
       end
       
-      def current_environment
-        if ENV.has_key?('RAILS_ENV')
-          ENV['RAILS_ENV']
-        elsif defined? Rails
-          Rails.env
-        end
-      end
-      
       def extract_enviroment_entry(all_configs, path)
-        env_str = current_environment
+        env_str = BackgroundQueue::Utils.current_environment
         if all_configs.has_key?(env_str)
           all_configs[env_str]
         elsif all_configs.has_key?(env_str.to_s.intern)

data/lib/background_queue/server_lib/balanced_queue.rb

@@ -17,10 +17,19 @@ module BackgroundQueue::ServerLib
     
     def add_task(task)
       @thread_manager.protect_access {
+        if task.replaced_while_waiting_to_retry?
+          @server.logger.debug("Not adding task that was replaced while waiting to retry (#{task.id})")
+          return
+        end
         status, existing_task = @task_registry.register(task)
         if status != :waiting
           if status == :existing
+            @server.logger.debug("Removing existing task (#{task.id})")
             remove_item(existing_task)
+          elsif status == :waiting_to_retry
+            @server.logger.debug("Removing existing task that is waiting to retry (#{task.id})")
+            existing_task.set_error_status(:replaced_while_waiting_to_retry)
+            finish_item(existing_task)
           end
           add_item(task)
           @thread_manager.signal_access #wake anything reading from the queue
@@ -36,6 +45,10 @@ module BackgroundQueue::ServerLib
     
     def finish_task(task)
       @thread_manager.protect_access {
+        if task.replaced_while_waiting_to_retry?
+          @server.logger.debug("Not finishing task that was replaced while waiting to retry (#{task.id})")
+          return
+        end
         finish_item(task)
         existing_task = @task_registry.de_register(task.id)
         if existing_task
@@ -44,6 +57,15 @@ module BackgroundQueue::ServerLib
       }
     end
     
+    #need to synchronise this...
+    def add_task_to_error_list(task)
+      @thread_manager.protect_access {
+        task.running = false
+        task.set_error_status(:waiting_to_retry)
+        @server.error_tasks.add_task(task)
+      }
+    end
+    
     def next_task
       task = nil
       @thread_manager.control_access {

data/lib/background_queue/server_lib/event_connection.rb

@@ -107,10 +107,16 @@ module BackgroundQueue::ServerLib
       for task_data in command.args[:tasks]
         if task_data[1].nil?
           merged_params = shared_params
+          merged_options = command.options
         else
           merged_params = shared_params.clone.update(task_data[1])
+          if task_data[2].nil?
+            merged_options = command.options
+          else
+            merged_options = command.options.merge(task_data[2])
+          end
         end
-        task = BackgroundQueue::ServerLib::Task.new(owner_id, job_id, task_data[0], priority, worker, merged_params, command.options)
+        task = BackgroundQueue::ServerLib::Task.new(owner_id, job_id, task_data[0], priority, worker, merged_params, merged_options)
         server.task_queue.add_task(task)
       end
       @server.change_stat(:tasks, command.args[:tasks].length)

data/lib/background_queue/server_lib/job.rb

@@ -19,6 +19,8 @@ module BackgroundQueue::ServerLib
     attr_reader :completed_weighted_tasks
     attr_reader :running_percent_weighted
     
+    attr_reader :summary
+    
     #attr_reader :current_running_excluded_status
     
     def initialize(id, owner)
@@ -69,7 +71,7 @@ module BackgroundQueue::ServerLib
         @total_weighted_tasks += 1
         @total_weighted_percent += task.weighted_percent
       end
-      @synchronous_count+=1 if task.synchronous?
+      #@synchronous_count+=1 if task.synchronous? #the queue only goes into sync mode once the task is running/about to run
       unless task.initial_progress_caption.nil? || task.initial_progress_caption.length == 0 || @current_progress[:percent] > 0
         @current_progress[:caption] = task.initial_progress_caption
       end
@@ -77,7 +79,10 @@ module BackgroundQueue::ServerLib
     end
     
     def next_item
-      pop
+      item = pop
+      @running_items += 1 if item
+      @synchronous_count+=1 if item && item.synchronous?
+      item
     end
     
     def remove_item(item)
@@ -85,16 +90,20 @@ module BackgroundQueue::ServerLib
     end
     
     def finish_item(item)
+      @running_items -= 1
       @synchronous_count-=1 if item.synchronous?
     end
     
     def synchronous?
-      @synchronous_count > 0
+      next_item = peek
+      @synchronous_count > 0 || (next_item && next_item.synchronous?)
     end
     
     def set_worker_status(status)
       if status[:meta]
         update_status_meta(status[:meta])
+      elsif status[:summary]
+        update_summary_meta(status)
       else
         running_status = get_running_status(status)
         if status[:percent] >= 100
@@ -138,19 +147,49 @@ module BackgroundQueue::ServerLib
     
     
     def update_status_meta(meta)
+      
       [:notice, :warning, :error].each { |key|
-        if meta[key]
+        val = BackgroundQueue::Utils.get_hash_entry(meta, key)
+        unless val.nil?
           @status_meta[key] = [] if @status_meta[key].nil?
-          @status_meta[key] << meta[key]
+          @status_meta[key] << val
         end
       }
-      if meta[:meta]
+      val = BackgroundQueue::Utils.get_hash_entry(meta, :meta)
+      unless val.nil?
         @status_meta[:meta] = {} if @status_meta[:meta].nil?
-        @status_meta[:meta] = @status_meta[:meta].update(meta[:meta])
+        @status_meta[:meta] = @status_meta[:meta].update(val)
       end
       update_current_progress
     end
     
+    def update_summary_meta(status)
+      @summary ||= {}
+      type = status[:type].intern
+      case status[:summary]
+      when "app"
+        @summary[type] ||= []
+        @summary[type] << status[:data]
+      when "set"
+        @summary[type] ||= {}
+        @summary[type][status[:key]] = status[:data]
+      when "inc"
+        @summary[type] ||= 0
+        @summary[type] += status[:data].to_i
+      when "dec"
+        @summary[type] ||= 0
+        @summary[type] -= status[:data].to_i
+      when "res"
+        if type == :all
+          @summary = {}
+        else
+          @summary.delete(type)
+        end
+      else
+        logger.error("Unknown summary action: #{status[:summary]}")
+      end
+    end
+    
     def update_finished_status(status)
       rstatus = deregister_running_status(status[:task_id])
       unless rstatus.nil?

data/lib/background_queue/server_lib/priority_queue.rb

@@ -8,6 +8,7 @@ module BackgroundQueue::ServerLib
       @items = {}
       @stalled_items = {}
       @stalled = false
+      @running_items = 0
     end
     
     def pop
@@ -53,6 +54,10 @@ module BackgroundQueue::ServerLib
       @queues.empty?
     end
     
+    def has_running_items?
+      @running_items > 0
+    end
+    
     def number_of_priorities
       @queues.length
     end
@@ -87,6 +92,7 @@ module BackgroundQueue::ServerLib
           return insert_queue_at_index(priority, idx)
         end
       end
+      return nil unless create
       return insert_queue_at_index(priority, -1)
     end
     
@@ -116,6 +122,7 @@ module BackgroundQueue::ServerLib
     attr_accessor :priority
     def initialize(priority)
       @priority = priority
+      raise "Invalid priority" if @priority.nil?
       super(0)
     end
     

data/lib/background_queue/server_lib/queue_registry.rb

@@ -9,6 +9,8 @@ module BackgroundQueue::ServerLib
           remove(queue, original_priority)
         end
         push(queue)
+      elsif queue.stalled? && !(queue.synchronous? && queue.has_running_items?) #it stalled because it was empty...
+        resume_queue(queue)
       end
     end
     
@@ -26,11 +28,15 @@ module BackgroundQueue::ServerLib
         else
           push(queue)
         end
+        @running_items += 1
+        item.running = true unless item.nil?
       end
+      #server.logger.debug("next item #{item.nil? ? 'nil' : item.id}")
       item
     end
     
     def remove_item(item)
+      item.running = false
       in_queue, queue = get_queue(get_queue_id_from_item(item), false)
       raise "Unable to remove task #{item.id} at priority #{item.priority} (no queue at that priority)" if queue.nil?
       priority_decreased, original_priority, item = remove_item_from_queue(queue, item)
@@ -48,7 +54,8 @@ module BackgroundQueue::ServerLib
       in_queue, queue = get_queue(get_queue_id_from_item(item), false)
       raise "Queue #{get_queue_id_from_item(item)} unavailble when finishing item" if queue.nil?
       queue.finish_item(item)
-      resume_queue(queue)
+      @running_items -= 1
+      resume_queue(queue) unless queue.synchronous? && queue.has_running_items?
     end
     
     private
@@ -94,21 +101,29 @@ module BackgroundQueue::ServerLib
     
     def stall_queue(queue)
       queue.stalled = true
+      server.logger.debug("stalling queue #{queue.id} (empty=#{queue.empty?})")
       #puts "stalling queue #{queue.inspect}"
       @stalled_items[queue.id] = queue
     end
     
     def resume_queue(queue)
       if queue.stalled?
-        @stalled_items.delete(queue.id) 
-        unless queue.empty? 
+        
+        if queue.empty? && !queue.has_running_items?
+          @stalled_items.delete(queue.id) 
+          @items.delete(queue.id)
+          server.logger.debug("removed empty queue #{queue.id}")
+          #puts "q empty"
+        elsif !queue.empty?
+          @stalled_items.delete(queue.id) 
           queue.stalled = false
           push(queue)
           @items[queue.id] = queue
+          server.logger.debug("resumed queue #{queue.id}")
           #puts "returned q: #{queue.inspect}"
         else
-          @items.delete(queue.id)
-          #puts "q empty"
+          server.logger.debug("keeping empty queue stalled #{queue.id}")
+          #keep stalled
         end
       #else
       #  puts "q not stalled"