backgroundrb-rails3 1.1

data/lib/backgroundrb/bdrb_job_queue.rb

@@ -0,0 +1,79 @@
+# Model for storing jobs/tasks persisted to the database
+
+class BdrbJobQueue < ActiveRecord::Base
+  validates_uniqueness_of :job_key,:scope => [:worker_name,:worker_key]
+  # find next task from the table
+  def self.find_next(worker_name,worker_key = nil)
+    returned_job = nil
+    ActiveRecord::Base.verify_active_connections!
+    transaction do
+      unless worker_key
+        #use ruby time stamps for time calculations as db might have different times than what is calculated by ruby/rails
+        t_job = find(:first,:conditions => [" worker_name = ? AND taken = ? AND scheduled_at <= ? ", worker_name, 0, Time.now.utc ],:lock => true, :order => 'priority desc')
+      else
+        t_job = find(:first,:conditions => [" worker_name = ? AND taken = ? AND worker_key = ? AND scheduled_at <= ? ", worker_name, 0, worker_key, Time.now.utc ],:lock => true)
+      end
+      if t_job
+        t_job.taken = 1
+        t_job.started_at = Time.now.utc
+        t_job.save
+        returned_job = t_job
+      end
+    end
+    returned_job
+  end
+
+  #these accessors get around any possible character encoding issues with the database
+  def args=(args)
+    write_attribute(:args, Base64.encode64(args))
+  end
+
+  def args
+    Base64.decode64(read_attribute(:args))
+  end
+
+  # release a job and mark it to be unfinished and free.
+  # useful, if inside a worker, processing of this job failed and you want it to process later
+  def release_job
+    ActiveRecord::Base.verify_active_connections!
+    self.class.transaction do
+      self.taken = 0
+      self.started_at = nil
+      self.save
+    end
+  end
+
+  # insert a new job for processing. jobs added will be automatically picked by the appropriate worker
+  def self.insert_job(options = { })
+    ActiveRecord::Base.verify_active_connections!
+    transaction do
+      options.merge!(:submitted_at => Time.now.utc,:finished => 0,:taken => 0)
+      t_job = new(options)
+      t_job.save
+    end
+  end
+
+  # remove a job from table
+  def self.remove_job(options = { })
+    ActiveRecord::Base.verify_active_connections!
+    transaction do
+      t_job_id = find(:first, :conditions => options.merge(:finished => 0,:taken => 0),:lock => true)
+      delete(t_job_id)
+    end
+  end
+
+  # Mark a job as finished
+  def finish!
+    ActiveRecord::Base.verify_active_connections!
+    self.class.transaction do
+      self.finished = 1
+      self.finished_at = Time.now.utc
+      self.job_key = "finished_#{Time.now.utc.to_i}_#{job_key}"
+      self.save
+    end
+    Thread.current[:persistent_job_id] = nil
+    Thread.current[:job_key] = nil
+    nil
+  end
+end
+

data/lib/backgroundrb/bdrb_result.rb

@@ -0,0 +1,19 @@
+module BackgrounDRb
+  class Result
+    def initialize results
+      @results = resuls
+    end
+
+    def async_response?
+      !(@results[:result] == true)
+    end
+
+    def sync_response?
+      (@results[:result] == true)
+    end
+
+    def error?
+      !(@results[:result_flag] == "ok")
+    end
+  end
+end

data/lib/backgroundrb/bdrb_start_stop.rb

@@ -0,0 +1,146 @@
+# see http://refspecs.freestandards.org/LSB_3.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html
+# for LSB-compliancy info
+module BackgrounDRb
+  class StartStop
+    def start
+      if running? # starting an already running process is considered a success
+        puts "BackgrounDRb Already Running"
+        exit(0)
+      elsif dead? # dead, but pid exists
+        remove_pidfiles
+      end
+
+      # status == 3, not running.
+      STDOUT.sync = true
+      print("Starting BackgrounDRb .... ")
+      start_bdrb
+      puts "Success!"
+    end
+
+    def stop
+      pid_files = Dir["#{RAILS_HOME}/tmp/pids/backgroundrb_*.pid"]
+      puts "BackgrounDRb Not Running" if pid_files.empty?
+      pid_files.each do |x|
+        begin
+          kill_process(x)
+        rescue Errno::ESRCH
+          # stopping an already stopped process is considered a success (exit status 0)
+        end
+      end
+      remove_pidfiles
+    end
+
+    # returns the correct lsb code for the status:
+    # 0 program is running or service is OK
+    # 1 program is dead and /var/run pid file exists
+    # 3 program is not running
+    def status
+      @status ||= begin
+                    if pidfile_exists? and process_running?
+                      0
+                    elsif pidfile_exists? # but not process_running
+                      1
+                    else
+                      3
+                    end
+                  end
+
+      return @status
+    end
+
+    def pidfile_exists?; File.exists?(PID_FILE); end
+
+    def process_running?
+      begin
+        Process.kill(0,self.pid)
+        true
+      rescue Errno::ESRCH
+        false
+      end
+    end
+
+    def running?;status == 0;end
+    # pidfile exists but process isn't running
+
+    def dead?;status == 1;end
+
+    def pid
+      File.read(PID_FILE).strip.to_i if pidfile_exists?
+    end
+
+    def remove_pidfiles
+      require 'fileutils'
+      FileUtils.rm_r(Dir["#{RAILS_HOME}/tmp/pids/backgroundrb_*.pid"])
+    end
+
+    def start_bdrb
+      require "rubygems"
+      require "yaml"
+      require "erb"
+      require "logger"
+      require "packet"
+      require "optparse"
+
+      require "bdrb_config"
+      require RAILS_HOME + "/config/boot"
+      require "active_support"
+
+      BackgrounDRb::Config.parse_cmd_options ARGV
+
+      require RAILS_HOME + "/config/environment"
+      require "bdrb_job_queue"
+      require "backgroundrb_server"
+
+      #proper way to daemonize - double fork to ensure parent can have no interest in the grandchild
+      if fork                     # Parent exits, child continues.
+        sleep(5)
+        exit(0)
+      else
+        Process.setsid                   # Become session leader.
+
+        op = File.open(PID_FILE, "w")
+        op.write(Process.pid().to_s)
+        op.close
+
+        if BDRB_CONFIG[:backgroundrb][:log].nil? or BDRB_CONFIG[:backgroundrb][:log] != 'foreground'
+          redirect_io(SERVER_LOGGER)
+        end
+        $0 = "backgroundrb master"
+        BackgrounDRb::MasterProxy.new()
+      end
+
+      #File.open(PID_FILE, "w") {|pidfile| pidfile.write(main_pid)}
+    end
+
+    def kill_process(pid_file)
+      pid = File.open(pid_file, "r") { |pid_handle| pid_handle.gets.strip.to_i }
+      pgid =  Process.getpgid(pid)
+      Process.kill('-TERM', pgid)
+      File.delete(pid_file) if File.exists?(pid_file)
+      puts "Stopped BackgrounDRb worker with pid #{pid}"
+    end
+
+
+    # Free file descriptors and
+    # point them somewhere sensible
+    # STDOUT/STDERR should go to a logfile
+    def redirect_io(logfile_name)
+      begin; STDIN.reopen "/dev/null"; rescue ::Exception; end
+
+      if logfile_name
+        begin
+          STDOUT.reopen logfile_name, "a"
+          STDOUT.sync = true
+        rescue ::Exception
+          begin; STDOUT.reopen "/dev/null"; rescue ::Exception; end
+        end
+      else
+        begin; STDOUT.reopen "/dev/null"; rescue ::Exception; end
+      end
+
+      begin; STDERR.reopen STDOUT; rescue ::Exception; end
+      STDERR.sync = true
+    end
+  end
+
+end

data/lib/backgroundrb/rails_worker_proxy.rb

@@ -0,0 +1,181 @@
+module BackgrounDRb
+  # A Worker proxy, which uses +method_missing+ for delegating method calls to the workers
+  class RailsWorkerProxy
+    attr_accessor :worker_name, :worker_method, :data, :worker_key,:middle_man
+
+    # create new worker proxy
+    def initialize(p_worker_name,p_worker_key = nil,p_middle_man = nil)
+      @worker_name = p_worker_name
+      @middle_man = p_middle_man
+      @worker_key = p_worker_key
+      @tried_connections = []
+    end
+
+    def method_missing(method_id,*args)
+      worker_method = method_id.to_s
+      arguments = args.first
+
+      arg,job_key,host_info,scheduled_at,priority = arguments && arguments.values_at(:arg,:job_key,:host,:scheduled_at, :priority)
+
+      # allow both arg and args
+      arg ||= arguments && arguments[:args]
+
+      new_schedule = (scheduled_at && scheduled_at.respond_to?(:utc)) ? scheduled_at.utc : Time.now.utc
+
+      if worker_method =~ /^async_(\w+)/
+        method_name = $1
+        worker_options = compact(:worker => worker_name,:worker_key => worker_key,
+                                 :worker_method => method_name,:job_key => job_key, :arg => arg)
+        run_method(host_info,:ask_work,worker_options)
+      elsif worker_method =~ /^enq_(\w+)/i
+        raise NoJobKey.new("Must specify a job key with enqueued tasks") if job_key.blank?
+        method_name = $1
+        marshalled_args = Marshal.dump(arg)
+        enqueue_task(compact(:worker_name => worker_name.to_s,:worker_key => worker_key.to_s,
+                             :worker_method => method_name.to_s,:job_key => job_key.to_s, :priority => priority,
+                             :args => marshalled_args,:timeout => arguments ? arguments[:timeout] : nil,:scheduled_at => new_schedule))
+       elsif worker_method =~ /^deq_(\w+)/i
+        raise NoJobKey.new("Must specify a job key to dequeue tasks") if job_key.blank?
+        method_name = $1
+        dequeue_task(compact(:worker_name => worker_name.to_s,:worker_key => worker_key.to_s,
+                             :worker_method => method_name.to_s,:job_key => job_key.to_s))
+      else
+        worker_options = compact(:worker => worker_name,:worker_key => worker_key,
+                                 :worker_method => worker_method,:job_key => job_key,:arg => arg)
+        run_method(host_info,:send_request,worker_options)
+      end
+    end
+
+    # enqueue tasks to the worker pool
+    def enqueue_task options = {}
+      BdrbJobQueue.insert_job(options)
+    end
+
+    # remove tasks from the worker pool
+    def dequeue_task options = {}
+      BdrbJobQueue.remove_job(options)
+    end
+
+    # invoke method on worker
+    def run_method host_info,method_name,worker_options = {}
+      result = []
+      connection = choose_connection(host_info)
+      raise NoServerAvailable.new("No BackgrounDRb server is found running") if connection.blank?
+      if host_info == :local or host_info.is_a?(String)
+        result << invoke_on_connection(connection,method_name,worker_options)
+      elsif host_info == :all
+        succeeded = false
+        begin
+          connection.each { |conn| result << invoke_on_connection(conn,method_name,worker_options) }
+          succeeded = true
+        rescue BdrbConnError; end
+        raise NoServerAvailable.new("No BackgrounDRb server is found running") unless succeeded
+      else
+        @tried_connections = [connection.server_info]
+        begin
+          result << invoke_on_connection(connection,method_name,worker_options)
+        rescue BdrbConnError => e
+          connection = middle_man.find_next_except_these(@tried_connections)
+          @tried_connections << connection.server_info
+          retry
+        end
+      end
+      #return nil if method_name == :ask_work
+      process_result(return_result(result))
+    end
+
+    def process_result t_result
+      case t_result
+      when Hash
+        if(t_result[:result] == true && t_result[:type] = :response)
+          if(t_result[:result_flag] == "ok")
+            return t_result[:data]
+          else
+            raise RemoteWorkerError.new("Error while executing worker method")
+          end
+        elsif(t_result[:result_flag] == "ok")
+          "ok"
+        elsif(t_result[:result_flag] == "error")
+          raise RemoteWorkerError.new("Error while executing worker method")
+        end
+      when Array
+        t_result
+      end
+    end
+
+    # choose a backgroundrb server connection and invoke worker method on it.
+    def invoke_on_connection connection,method_name,options = {}
+      raise NoServerAvailable.new("No BackgrounDRb is found running") unless connection
+      connection.send(method_name,options)
+    end
+
+    # get results back from the cache. Cache can be in-memory worker cache or memcache
+    # based cache
+    def ask_result job_key
+      options = compact(:worker => worker_name,:worker_key => worker_key,:job_key => job_key)
+      if BDRB_CONFIG[:backgroundrb][:result_storage] == 'memcache'
+        return_result_from_memcache(options)
+      else
+        result = middle_man.backend_connections.map { |conn| conn.ask_result(options) }
+        return_result(result)
+      end
+    end
+
+    # return runtime information about worker
+    def worker_info
+      t_connections = middle_man.backend_connections
+      result = t_connections.map { |conn| conn.worker_info(compact(:worker => worker_name,:worker_key => worker_key)) }
+      return_result(result)
+    end
+
+    # generate worker key
+    def gen_key options
+      key = [options[:worker],options[:worker_key],options[:job_key]].compact.join('_')
+      key
+    end
+
+    # return result from memcache
+    def return_result_from_memcache options = {}
+      middle_man.cache[gen_key(options)]
+    end
+
+    # reset result within memcache for given key
+    def reset_memcache_result(job_key,value)
+      options = compact(:worker => worker_name,:worker_key => worker_key,\
+                          :job_key => job_key)
+      key = gen_key(options)
+      middle_man.cache[key] = value
+      value
+    end
+
+    def return_result result
+      result = Array(result)
+      result.size <= 1 ? result[0] : result
+    end
+
+    # delete a worker
+    def delete
+      middle_man.backend_connections.each do |connection|
+        connection.delete_worker(compact(:worker => worker_name, :worker_key => worker_key))
+      end
+      return worker_key
+    end
+
+    # choose a worker
+    def choose_connection host_info
+      case host_info
+      when :all; middle_man.backend_connections
+      when :local; middle_man.find_local
+      when String; middle_man.find_connection(host_info)
+      else; middle_man.choose_server
+      end
+    end
+
+    # helper method to compact a hash and for getting rid of nil parameters
+    def compact(options = { })
+      options.delete_if { |key,value| value.nil? }
+      options
+    end
+  end # end of RailsWorkerProxy class
+
+end # end of BackgrounDRb module

data/lib/backgroundrb/railtie.rb

@@ -0,0 +1,48 @@
+# Backgroundrb
+# FIXME: check if data that we are writing to the socket should end with newline
+require "pathname"
+require "packet"
+require "ostruct"
+
+require "rails"
+require "backgroundrb"
+
+require "backgroundrb/bdrb_config"
+require "backgroundrb/bdrb_client_helper"
+require "backgroundrb/bdrb_job_queue"
+require "backgroundrb/bdrb_conn_error"
+require "backgroundrb/rails_worker_proxy"
+require "backgroundrb/bdrb_connection"
+require "backgroundrb/bdrb_cluster_connection"
+require "backgroundrb/bdrb_start_stop"
+require "backgroundrb/bdrb_result"
+
+
+module BackgrounDRb
+  class Railtie < Rails::Railtie
+    config.bdrb = ActiveSupport::OrderedOptions.new
+
+    unless config.bdrb.has_key? :root
+      config.bdrb.root = File.expand_path(File.join(__FILE__, '..', '..', '..'))
+    end
+    BackgrounDRb::BACKGROUNDRB_ROOT = config.bdrb.root
+
+    config.before_configuration do
+      config_file = "#{Rails.root}/config/backgroundrb.yml"
+
+      if File.exists?(config_file) && !config.bdrb.has_key?(:config)
+        config.bdrb.config = BackgrounDRb::Config.read_config(config_file)
+      end
+      BackgrounDRb::BDRB_CONFIG = config.bdrb.config
+
+      MiddleMan = BackgrounDRb::ClusterConnection.new if File.exists?(config_file)
+    end
+
+    rake_tasks do
+      load "tasks/backgroundrb_tasks.rake"
+    end
+
+  end
+
+end
+