background_queue 0.3.0 → 0.6.0

data/lib/background_queue/server_lib/task.rb

@@ -18,6 +18,7 @@ module BackgroundQueue::ServerLib
       @job_id = job_id
       @id = id
       @priority = priority
+      raise "Missing priority" if @priority.nil?
       @worker = worker
       @running = false
       @options = options
@@ -48,6 +49,10 @@ module BackgroundQueue::ServerLib
       @job = job
     end
     
+    def get_job
+      @job
+    end
+    
     def is_excluded_from_count?
       @options[:exclude] == true
     end
@@ -68,6 +73,10 @@ module BackgroundQueue::ServerLib
       @options[:initial_progress_caption]
     end
     
+    def send_summary?
+      @options[:send_summary] 
+    end
+    
     def get_error_count
       if @error_count.nil?
         0
@@ -76,11 +85,30 @@ module BackgroundQueue::ServerLib
       end
     end
     
+    def set_error_status(e_status)
+      @error_status = e_status
+    end
+    
+    def get_error_status
+      @error_status
+    end
+    
+    def waiting_to_retry?
+      @error_status == :waiting_to_retry
+    end
+    
+    def replaced_while_waiting_to_retry?
+      @error_status == :replaced_while_waiting_to_retry
+    end
     
     def increment_error_count
       @error_count = get_error_count + 1
     end
     
+    def step
+      @options[:step]
+    end
+    
     
     def set_worker_status(status)
       raise "Task without job set" if @job.nil?

data/lib/background_queue/server_lib/task_registry.rb

@@ -11,12 +11,19 @@ module BackgroundQueue::ServerLib
     def register(task)
       existing_task = @tasks[task.id]
       if existing_task.nil? 
+        #puts "nil task"
         @tasks[task.id] = task
         [:new, nil]
       elsif existing_task.running?
+        #puts "task running"
         register_waiting_task(task)
         [:waiting, nil]
+      elsif existing_task.waiting_to_retry?
+        #puts "task waiting_to_retry"
+        @tasks[task.id] = task
+        [:waiting_to_retry, existing_task]
       else
+        #puts "task waiting"
         @tasks[task.id] = task
         [:existing, existing_task]
       end

data/lib/background_queue/server_lib/worker_client.rb

@@ -21,7 +21,7 @@ module BackgroundQueue::ServerLib
         end
         return :ok
       rescue BackgroundQueue::ServerLib::WorkerError => we
-        @server.logger.error("Worker Error sending request #{task.id} to worker: #{e.message}")
+        @server.logger.error("Worker Error sending request #{task.id} to worker: #{we.message}")
         return :worker_error
       rescue BackgroundQueue::ServerLib::ThreadManager::ForcedStop => fe
         @server.logger.error("Thread stop while sending request #{task.id} to worker: #{fe.message}")
@@ -37,7 +37,10 @@ module BackgroundQueue::ServerLib
     
     def build_request(uri, task, secret)
       req = Net::HTTP::Post.new(uri.path)
-      req.set_form_data({:task=>task.to_json, :auth=>secret, :server_port=>@server.config.address.port})
+      form_data = {:task=>task.to_json, :auth=>secret, :server_port=>@server.config.address.port}
+      form_data[:summary] = task.get_job.summary.to_json if task.send_summary? && !task.get_job.summary.nil?
+      form_data[:step] = task.step unless task.step.nil?
+      req.set_form_data(form_data)
       req["host"] = task.domain
       req
     end

data/lib/background_queue/server_lib/worker_thread.rb

@@ -42,7 +42,7 @@ module BackgroundQueue::ServerLib
           else
             @server.logger.debug("failed calling worker for task #{task.id}")
             @server.workers.finish_using_worker(worker, result == :worker_error)
-            @server.error_tasks.add_task(task)
+            @server.task_queue.add_task_to_error_list(task)
             return result != :stop
             #error_count += 1
             #if error_count > 5

data/lib/background_queue/utils.rb

@@ -2,6 +2,22 @@ module BackgroundQueue
   #Utility Module
   module Utils
     
+    def self.current_environment
+      if ENV.has_key?('RAILS_ENV')
+        ENV['RAILS_ENV']
+      elsif defined? Rails
+        Rails.env
+      end
+    end
+    
+    def self.current_root
+      if defined? RAILS_ROOT
+        RAILS_ROOT
+      elsif defined? Rails
+        Rails.root
+      end
+    end
+    
     #gets an entry from a hash regardless if the key is a string or symbol
     def self.get_hash_entry(hash, key)
       if hash.has_key?(key)
@@ -20,7 +36,11 @@ module BackgroundQueue
       
       #wrap a hash
       def initialize(hash)
-        @hash = hash
+        if hash.kind_of?(Hash)
+          @hash = hash
+        else
+          raise "Invalid class used when initializing AnyKeyHash (#{hash.class.name})"
+        end
       end
       
       #get an entry by string or symbol
@@ -35,6 +55,10 @@ module BackgroundQueue
       def to_json(dummy=true)
         @hash.to_json
       end
+      
+      def merge(other_map)
+        AnyKeyHash.new(@hash.clone.update(other_map))
+      end
     end
   end
 end

data/lib/background_queue/worker/base.rb

@@ -19,6 +19,7 @@ module BackgroundQueue::Worker
     def set_progress(caption, percent)
       #puts self
       #puts "env=#{self.environment}"
+      logger.debug("set_progress(#{caption}, #{percent})")
       self.environment.send_data({:caption=>caption, :percent=>percent}.to_json)
     end
     
@@ -26,14 +27,50 @@ module BackgroundQueue::Worker
     #key: :notice, :warning, :error, :meta
     #value: :notice/:warning/:error : String, :meta : any json compatible object
     def add_progress_meta(key, value)
+      logger.debug("add_progress_meta(#{key}, #{value.to_json})")
       self.environment.send_data({:meta=>{key=>value}}.to_json)
     end
     
+    def append_summary(type, data)
+      raise "Missing Type when appending summary" if type.nil?
+      self.environment.send_data({:summary=>"app", :type=>type.to_s, :data=>data}.to_json)
+    end
+    
+    def set_summary(type, key, data)
+      raise "Missing Type when settind summary" if type.nil?
+      raise "Missing key when settind summary" if key.nil?
+      self.environment.send_data({:summary=>"set", :type=>type.to_s, :key=>key, :data=>data}.to_json)
+    end
+    
+    def increment_summary(type, amount=1)
+      raise "Missing Type when incrementing summary" if type.nil?
+      self.environment.send_data({:summary=>"inc", :type=>type.to_s, :data=>amount}.to_json)
+    end
+    
+    def decrement_sumary(type, amount=1)
+      raise "Missing Type when decrementing summary" if type.nil?
+      self.environment.send_data({:summary=>"dec", :type=>type.to_s, :data=>amount}.to_json)
+    end
+    
+    def reset_summary(type)
+      self.environment.send_data({:summary=>"res", :type=>type.to_s}.to_json)
+    end
+    
     #virtual function: called to process a worker request
     def run
       raise "run() Not Implemented on worker #{self.class.name}"
     end
     
+    #virtual function: called to process a worker request (step=start)
+    def start
+      raise "start() Not Implemented on worker #{self.class.name}"
+    end
+    
+    #virtual function: called to process a worker request (step=finish)
+     def finish
+      raise "finish() Not Implemented on worker #{self.class.name}"
+    end
+    
     def params
       self.environment.params
     end
@@ -42,5 +79,9 @@ module BackgroundQueue::Worker
       self.environment.logger
     end
     
+    def summary
+      self.environment.summary
+    end
+    
   end
 end

data/lib/background_queue/worker/calling.rb

@@ -34,17 +34,40 @@ module BackgroundQueue::Worker
       env
     end
     
+    def set_process_name(env)
+      unless BackgroundQueue::Worker::Config.process_name_prefix.nil?
+        $0 = "#{BackgroundQueue::Worker::Config.process_name_prefix}:#{env.worker}:#{env.owner_id}:#{env.job_id}:#{env.task_id}"
+      end
+    end
+    
+    def revert_process_name
+      unless BackgroundQueue::Worker::Config.process_name_prefix.nil?
+        $0 = "#{BackgroundQueue::Worker::Config.process_name_prefix}:idle"
+      end
+    end
+    
     def call_worker(worker, env)
       headers['X-Accel-Buffering'] = 'no' #passenger standalone uses nginx. This will turn buffering off in nginx
       render :text => lambda { |response,output| 
         env.set_output(output)
+        set_process_name(env)
         begin
-          worker.run
+          case env.step
+          when "start"
+            worker.start
+          when "finish"
+            worker.finish
+          else
+            worker.run
+          end
         rescue Exception=>e
           logger.error("Error calling worker: #{e.message}")
           logger.error(e.backtrace.join("\n"))
+          worker.logger.error("Error calling worker: #{e.message}")
+          worker.logger.error(e.backtrace.join("\n"))
         ensure
           worker.set_environment(nil)
+          revert_process_name
         end
       }, :type=>"text/text"
     end

data/lib/background_queue/worker/config.rb

@@ -2,6 +2,8 @@ module BackgroundQueue::Worker
   class Config
       
     @@worker_path = nil
+    @separate_logs = false
+    @@process_name_prefix = "bgq"
     
     def self.secret=(auth)
       @@secret = auth
@@ -33,5 +35,23 @@ module BackgroundQueue::Worker
       end
     end
     
+    def self.separate_logs=(use_separate_logs)
+      @separate_logs = use_separate_logs
+    end
+    
+    #should the worker have its own log file per instance? (makes it easier to backtrack errors)
+    #if true, the log file is based on the job id. Each log line will have the task id.
+    def self.separate_logs?
+      @separate_logs
+    end
+    
+    
+    def self.process_name_prefix=(prefix)
+      @@process_name_prefix = prefix
+    end
+    
+    def self.process_name_prefix
+      @@process_name_prefix
+    end
   end
 end

data/lib/background_queue/worker/environment.rb

@@ -9,6 +9,8 @@ module BackgroundQueue::Worker
     attr_reader :owner_id
     attr_reader :worker
     attr_reader :logger
+    attr_reader :summary
+    attr_reader :step
     
     attr_reader :server_address
     
@@ -20,8 +22,12 @@ module BackgroundQueue::Worker
     
     def init_from_controller(controller)
       @controller = controller
-      @logger = controller.logger
       init_params(controller.params)
+      if BackgroundQueue::Worker::Config.separate_logs?
+        @logger = BackgroundQueue::Worker::Logger.init_logger(@worker, @owner_id, @job_id , @task_id, controller.logger.level)
+      else
+        @logger = controller.logger
+      end   
       init_server_address(controller)
     end
     
@@ -40,6 +46,20 @@ module BackgroundQueue::Worker
       @task_id = hash_data['id']
       @priority = hash_data['priority']
       @worker = hash_data['worker']
+      
+      
+      summary_data = nil
+      begin
+        summary_data = JSON.load(controller_params[:summary]) unless controller_params[:summary].nil?
+      rescue Exception=>e
+        raise "Invalid data format (should be json) when loading summary from buffer: #{e.message}"
+      end
+      if summary_data.nil?
+        @summary = {}
+      else
+        @summary = BackgroundQueue::Utils::AnyKeyHash.new(summary_data)
+      end
+      @step = controller_params[:step]
     end
     
     def set_output(out)
@@ -55,6 +75,10 @@ module BackgroundQueue::Worker
       @server_address = BackgroundQueue::Worker::Environment::Server.new(controller.request.remote_ip, controller.params[:server_port])
     end
     
+    def revert_environment
+      
+    end
+    
     class Server
       attr_accessor :host
       attr_accessor :port

data/lib/background_queue/worker/logger.rb

@@ -0,0 +1,114 @@
+module BackgroundQueue::Worker
+  class Logger < Logger
+  
+    def initialize(logdev, task_key)
+			super(logdev, 0, 1048576)
+			@default_formatter = BackgroundQueue::Worker::LogFormatter.new(task_key)
+		end
+		
+		def set_previous_state(previous_state)
+		  @previous_state = previous_state
+		end
+			
+		def format_message(severity, datetime, progname, msg)
+			(@formatter || @default_formatter).call(severity, datetime, progname, msg)
+		end
+		
+		def self.init_logger(worker_name, owner_key, job_key, task_key, level)
+		  logger = build_logger("#{BackgroundQueue::Utils.current_root}/log/workers/#{worker_name}-#{owner_key}-#{job_key}.log", task_key, level)
+		  
+		  prev_state = {}
+		  
+		  #we also want to have active record use this log file
+		  if defined? ActiveRecord && defined? ActiveRecord::Base && ActiveRecord::Base.class.respond_to?(:logger)
+		    begin
+          prev_state[:ar_base] = ActiveRecord::Base.logger
+          ActiveRecord::Base.logger = logger
+          
+          if ActiveRecord::Base.respond_to?(:active_connections)
+            #set the logger for connections that are already loaded
+            ActiveRecord::Base.active_connections.each_value { |con| con.set_logger(ActiveRecord::Base.logger)}
+          end
+        rescue Exception=>e
+          logger.debug("Error setting up active record logger: #{e.message}")
+        end
+		  end
+		  
+		  #and RAILS_DEFAULT_LOGGER
+		  if defined?(RAILS_DEFAULT_LOGGER)
+		    prev_state[:rails_default_logger] = RAILS_DEFAULT_LOGGER
+        Object.redefine_const(
+          :RAILS_DEFAULT_LOGGER,
+          logger
+        )
+		  end
+      
+		  logger.set_previous_state(prev_state) unless prev_state.empty?
+      logger
+		end
+		
+		def self.build_logger(path, task_key, level)
+		  logger = BackgroundQueue::Worker::Logger.new(path, task_key)
+		  logger.level = level
+		  logger
+		end
+		
+		def revert_to_previous_state
+		  unless @previous_state.nil?
+		    if @previous_state[:ar_base]
+		      ActiveRecord::Base.logger = @previous_state[:ar_base]
+		      if ActiveRecord::Base.respond_to?(:active_connections)
+            #set the logger for connections that are already loaded
+            ActiveRecord::Base.active_connections.each_value { |con| con.set_logger(ActiveRecord::Base.logger)}
+          end
+		    end
+		    if @previous_state[:rails_default_logger]
+		      Object.redefine_const(
+            :RAILS_DEFAULT_LOGGER,
+            @previous_state[:rails_default_logger]
+          )
+		    end
+		  end
+		end
+  end
+  
+  
+  #custom log formatter
+	class LogFormatter < Logger::Formatter
+		Format = "%s %5s [%s] %s\n"
+		
+		def initialize(task_key)
+      @datetime_format = "%Y-%m-%d %H:%M:%S"
+      @task_key = task_key
+    end
+		
+		def call(severity, time, progname, msg)
+      Format % [@task_key, severity, format_datetime(time), msg2str(msg)]
+    end
+	end
+end
+
+#allow us to redifine constants (RAILS_DEFAULT_LOGGER)
+unless  Module.respond_to?(:redefine_const)
+  class Module
+    def redefine_const(name, value)
+      __send__(:remove_const, name) if const_defined?(name)
+      const_set(name, value)
+    end
+  end
+end
+
+
+#need to be able to set the current logger for existing connections...
+if defined? ActiveRecord && defined? ActiveRecord::ConnectionAdapters && defined? ActiveRecord::ConnectionAdapters::AbstractAdapter
+  module ActiveRecord
+    module ConnectionAdapters # :nodoc:
+      class AbstractAdapter
+        def set_logger(logger)
+          @logger = logger
+        end
+      end
+    end
+  end
+end
+