backgroundrb-rails3 1.1

data/script/load_worker_env.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+RAILS_HOME = File.expand_path(File.join(File.dirname(__FILE__),".."))
+
+require "rubygems"
+require "active_support"
+require "active_record"
+
+require "yaml"
+require "erb"
+require "logger"
+require "optparse"
+
+require RAILS_HOME + "/config/boot"
+require "backgroundrb"
+
+BDRB_HOME = ::BackgrounDRb::BACKGROUNDRB_ROOT
+
+["server","server/lib","lib","lib/backgroundrb"].each { |x| $LOAD_PATH.unshift(BDRB_HOME + "/#{x}")}
+
+$LOAD_PATH.unshift(File.join(RAILS_HOME,"lib","workers"))
+
+require "bdrb_config"
+
+BDRB_CONFIG = BackgrounDRb::Config.read_config("#{RAILS_HOME}/config/backgroundrb.yml")
+
+if !(::Packet::WorkerRunner::WORKER_OPTIONS[:worker_env] == false)
+  require RAILS_HOME + "/config/environment"
+end
+require "backgroundrb_server"
+

data/script/monitrc

@@ -0,0 +1,25 @@
+set daemon 60
+set mailserver localhost
+set mail-format {
+  from: sample@foobar.com
+  subject: Alert from Backgroundrb
+}
+set alert hemant@gmail.com
+
+set httpd port 3000 and
+    use address localhost  # only accept connection from localhost
+    allow localhost        # allow localhost to connect to the server and
+
+check process backgroundrb
+    with pidfile /home/hemant/rails_sandbox/tmp/pids/backgroundrb_11008.pid
+    start program = "/usr/bin/ruby /home/hemant/rails_sandbox/script/backgroundrb start"
+    stop program  = "/usr/bin/ruby /home/hemant/rails_sandbox/script/backgroundrb stop"
+    if totalmem > 50.0 MB for 5 cycles then restart
+    if cpu usage > 95% for 3 cycles then restart
+
+    if failed port 11008 type tcp
+        with timeout 30 seconds
+        for 5 cycles
+        then restart
+    group backgroundrb
+

data/server/backgroundrb_server.rb

@@ -0,0 +1,12 @@
+require "chronic"
+require "lib/bdrb_result_storage"
+require "lib/bdrb_thread_pool"
+require "lib/bdrb_server_helper"
+require "lib/master_worker"
+require "lib/master_proxy"
+require "lib/cron_trigger"
+require "lib/invalid_dump_error"
+require "lib/log_worker"
+require "lib/trigger"
+require "lib/meta_worker"
+

data/server/lib/bdrb_result_storage.rb

@@ -0,0 +1,62 @@
+module BackgrounDRb
+  class ResultStorage
+    attr_accessor :cache,:worker_name,:worker_key,:storage_type
+    def initialize(worker_name,worker_key,storage_type = nil)
+      @worker_name = worker_name
+      @worker_key = worker_key
+      @mutex = Mutex.new
+      @storage_type = storage_type
+      @cache = (@storage_type == 'memcache') ? memcache_instance : {}
+    end
+
+    # Initialize Memcache for result or object caching
+    def memcache_instance
+      require 'memcache'
+      memcache_options = {
+        :c_threshold => 10_000,
+        :compression => true,
+        :debug => false,
+        :namespace => 'backgroundrb_result_hash',
+        :readonly => false,
+        :urlencode => false
+      }
+      t_cache = MemCache.new(memcache_options)
+      t_cache.servers = BDRB_CONFIG[:memcache].split(',')
+      t_cache
+    end
+
+    # generate key based on worker_name and worker_key
+    # for local cache, there is no need of unique key
+    def gen_key key
+      if storage_type == 'memcache'
+        key = [worker_name,worker_key,key].compact.join('_')
+        key
+      else
+        key
+      end
+    end
+
+    # fetch object from cache
+    def [] key
+      @mutex.synchronize { @cache[gen_key(key)] }
+    end
+
+    def []= key,value
+      @mutex.synchronize { @cache[gen_key(key)] = value }
+    end
+
+    def delete key
+      @mutex.synchronize { @cache.delete(gen_key(key)) }
+    end
+
+    def shift key
+      val = nil
+      @mutex.synchronize do
+        val = @cache[key]
+        @cache.delete(key)
+      end
+      return val
+    end
+  end
+end
+

data/server/lib/bdrb_server_helper.rb

@@ -0,0 +1,24 @@
+module BackgrounDRb
+  module BdrbServerHelper
+    # Load data using Marshal.load, if load fails because of undefined constant
+    # try to load the constant. FIXME: regexp needs to handle all the cases.
+    def load_data data
+      begin
+        return Marshal.load(data)
+      rescue
+        error_msg = $!.message
+        if error_msg =~ /^undefined\ .+\ ([A-Z][^:]+)/
+          file_name = $1.underscore
+          begin
+            require file_name
+            return Marshal.load(data)
+          rescue
+            return nil
+          end
+        else
+          return nil
+        end
+      end # end of load_data method
+    end
+  end
+end

data/server/lib/bdrb_thread_pool.rb

@@ -0,0 +1,127 @@
+module BackgrounDRb
+
+  class InterruptedException < RuntimeError ; end
+
+  class WorkData
+    attr_accessor :args,:block,:job_method,:persistent_job_id,:job_key
+    def initialize(args,job_key,job_method,persistent_job_id)
+      @args = args
+      @job_key = job_key
+      @job_method = job_method
+      @persistent_job_id = persistent_job_id
+    end
+  end
+
+  class ThreadPool
+    attr_accessor :size,:threads,:work_queue,:logger
+    attr_accessor :result_queue,:master
+
+    def initialize(master,size,logger)
+      @master = master
+      @logger = logger
+      @size = size
+      @threads = []
+      @work_queue = []
+      @mutex = Monitor.new
+      @cv = @mutex.new_cond
+      @size.times { add_thread }
+    end
+
+    # can be used to make a call in threaded manner
+    # passed block runs in a thread from thread pool
+    # for example in a worker method you can do:
+    #   def user_tags url
+    #     thread_pool.defer(:fetch_url,url)
+    #   end
+    #   def fetch_url(url)
+    #     begin
+    #       data = Net::HTTP.get(url,'/')
+    #       File.open("#{RAILS_ROOT}/log/pages.txt","w") do |fl|
+    #         fl.puts(data)
+    #       end
+    #     rescue
+    #       logger.info "Error downloading page"
+    #     end
+    #   end
+    # you can invoke above method from rails as:
+    #   MiddleMan.worker(:rss_worker).async_user_tags(:arg => "en.wikipedia.org")
+    # assuming method is defined in rss_worker
+
+    def defer(method_name,args = nil)
+      @mutex.synchronize do
+        job_key = Thread.current[:job_key]
+        persistent_job_id = Thread.current[:persistent_job_id]
+        @cv.wait_while { @work_queue.size >= size }
+        @work_queue.push(WorkData.new(args,job_key,method_name,persistent_job_id))
+        @cv.broadcast
+      end
+    end
+
+    # Start worker threads
+    def add_thread
+      @threads << Thread.new do
+        Thread.current[:job_key] = nil
+        Thread.current[:persistent_job_id] = nil
+        while true
+          begin
+            task = nil
+            @mutex.synchronize do
+              @cv.wait_while { @work_queue.size == 0 }
+              task = @work_queue.pop
+              @cv.broadcast
+            end
+            if task
+              Thread.current[:job_key] = task.job_key
+              Thread.current[:persistent_job_id] = task.persistent_job_id
+              block_result = run_task(task)
+            end
+          rescue BackgrounDRb::InterruptedException
+            STDERR.puts("BackgrounDRb thread interrupted: #{Thread.current.inspect}")
+            STDERR.flush
+          end
+        end
+      end
+    end
+
+    # run tasks popped out of queue
+    def run_task task
+      block_arity = master.method(task.job_method).arity
+      begin
+        check_db_connection
+        t_data = task.args
+        result = nil
+        if block_arity != 0
+          result = master.send(task.job_method,task.args)
+        else
+          result = master.send(task.job_method)
+        end
+        return result
+      rescue BackgrounDRb::InterruptedException => e
+        # Don't log, just re-raise
+        raise e
+      rescue Object => bdrb_error
+        log_exception(bdrb_error)
+        return nil
+      end
+    end
+
+    def log_exception exception_object
+      STDERR.puts exception_object.to_s
+      STDERR.puts exception_object.backtrace.join("\n")
+      STDERR.flush
+    end
+
+
+    # Periodic check for lost database connections and closed connections
+    def check_db_connection
+      begin
+        ActiveRecord::Base.verify_active_connections! if defined?(ActiveRecord)
+      rescue Object => bdrb_error
+        log_exception(bdrb_error)
+      end
+    end
+
+
+  end #end of class ThreadPool
+end # end of module BackgrounDRb
+

data/server/lib/cron_trigger.rb

@@ -0,0 +1,197 @@
+module BackgrounDRb
+  class CronTrigger
+    WDAYS = { 0 => "Sunday",1 => "Monday",2 => "Tuesday",3 => "Wednesday", 4 => "Thursday", 5 => "Friday", 6 => "Saturday" }
+    LeapYearMonthDays = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+    CommonYearMonthDays = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+
+    attr_reader :sec, :min, :hour, :day, :month, :wday, :year, :cron_expr
+
+    # initialize the Cron Trigger
+    def initialize(expr)
+      self.cron_expr = expr
+    end
+
+    # create the cron expression and populate instance variables.
+    def cron_expr=(expr)
+      @cron_expr = expr
+      self.sec, self.min, self.hour, self.day, self.month, self.wday, self.year = @cron_expr.split(' ')
+    end
+
+    def fire_after_time(p_time)
+      @t_sec,@t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday,@t_yday,@t_idst,@t_zone = p_time.to_a
+      @count = 0
+      loop do
+        @count += 1
+
+        if @year && !@year.include?(@t_year)
+          return nil if @t_year > @year.max
+          @t_year = @year.detect { |y| y > @t_year }
+        end
+
+        # if range of months doesn't include current month, find next month from the range
+        unless @month.include?(@t_month)
+          next_month = @month.detect { |m| m > @t_month } || @month.min
+          @t_day,@t_hour,@t_min,@t_sec = @day.min,@hour.min,@min.min,@sec.min
+          if next_month < @t_month
+            @t_month = next_month
+            @t_year += 1
+            retry
+          end
+          @t_month = next_month
+        end
+
+        if !day_restricted? && wday_restricted?
+          unless @wday.include?(@t_wday)
+            next_wday = @wday.detect { |w| w > @t_wday} || @wday.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+            t_time = Chronic.parse("next #{WDAYS[next_wday]}",:now => current_time)
+            @t_day,@t_month,@t_year = t_time.to_a[3..5]
+            @t_wday = next_wday
+            retry
+          end
+        elsif !wday_restricted? && day_restricted?
+          day_range = (1.. month_days(@t_year,@t_month))
+          # day array, that includes days which are present in current month
+          day_array = @day.select { |d| day_range === d }
+          unless day_array.include?(@t_day)
+            next_day = day_array.detect { |d| d > @t_day } || day_array.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+            if !next_day || next_day < @t_day
+              t_time = Chronic.parse("next month",:now => current_time)
+              @t_day = next_day.nil? ? @day.min : next_day
+              @t_month,@t_year = t_time.month,t_time.year
+              retry
+            end
+            @t_day = next_day
+          end
+        else
+          # if both day and wday are restricted cron should give preference to one thats closer to current time
+          day_range = (1 .. month_days(@t_year,@t_month))
+          day_array = @day.select { |d| day_range === d }
+          if !day_array.include?(@t_day) && !@wday.include?(@t_wday)
+            next_day = day_array.detect { |d| d > @t_day } || day_array.min
+            next_wday = @wday.detect { |w| w > @t_wday } || @wday.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+
+            # if next_day is nil or less than @t_day it means that it should run in next month
+            if !next_day || next_day < @t_day
+              next_time_mday = Chronic.parse("next month",:now => current_time)
+            else
+              @t_day = next_day
+              next_time_mday = current_time
+            end
+            next_time_wday = Chronic.parse("next #{WDAYS[next_wday]}",:now => current_time)
+            if next_time_mday < next_time_wday
+              @t_day,@t_month,@t_year = next_time_mday.to_a[3..5]
+            else
+              @t_day,@t_month,@t_year = next_time_wday.to_a[3..5]
+            end
+            retry
+          end
+        end
+
+        unless @hour.include?(@t_hour)
+          next_hour = @hour.detect { |h| h > @t_hour } || @hour.min
+          @t_min,@t_sec = @min.min,@sec.min
+          if next_hour < @t_hour
+            @t_hour = next_hour
+            next_day = Chronic.parse("next day",:now => current_time)
+            @t_day,@t_month,@t_year,@t_wday = next_day.to_a[3..6]
+            retry
+          end
+          @t_hour = next_hour
+        end
+
+        unless @min.include?(@t_min)
+          next_min = @min.detect { |m| m > @t_min } || @min.min
+          @t_sec = @sec.min
+          if next_min < @t_min
+            @t_min = next_min
+            next_hour = Chronic.parse("next hour",:now => current_time)
+            @t_hour,@t_day,@t_month,@t_year,@t_wday = next_hour.to_a[2..6]
+            retry
+          end
+          @t_min = next_min
+        end
+
+        unless @sec.include?(@t_sec)
+          next_sec = @sec.detect { |s| s > @t_sec } || @sec.min
+          if next_sec < @t_sec
+            @t_sec = next_sec
+            next_min = Chronic.parse("next minute",:now => current_time)
+            @t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday = next_min.to_a[1..6]
+            retry
+          end
+          @t_sec = next_sec
+        end
+        break
+      end # end of loop do
+      current_time
+    end
+
+    def current_time
+      Time.local(@t_sec,@t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday,nil,@t_idst,@t_zone)
+    end
+
+    def day_restricted?
+      return !@day.eql?(1..31)
+    end
+
+    def wday_restricted?
+      return !@wday.eql?(0..6)
+    end
+
+    # TODO: mimic attr_reader to define all of these
+    def sec=(sec); @sec = parse_part(sec, 0 .. 59); end
+
+    def min=(min); @min = parse_part(min, 0 .. 59); end
+
+    def hour=(hour); @hour = parse_part(hour, 0 .. 23); end
+
+    def day=(day)
+      @day = parse_part(day, 1 .. 31)
+    end
+
+    def month=(month)
+      @month = parse_part(month, 1 .. 12)
+    end
+
+    def year=(year)
+      @year = parse_part(year)
+    end
+
+    def wday=(wday)
+      @wday = parse_part(wday, 0 .. 6)
+    end
+    private
+    def month_days(y, m)
+      if ((y % 4 == 0) && (y % 100 != 0)) || (y % 400 == 0)
+        LeapYearMonthDays[m-1]
+      else
+        CommonYearMonthDays[m-1]
+      end
+    end
+
+    # 0-5,8,10; 0-5; *; */5
+    def parse_part(part, range=nil)
+      return range  if part.nil? or part == '*' or part =~ /^[*0]\/1$/
+
+      r = Array.new
+      part.split(',').each do |p|
+        if p =~ /-/  # 0-5
+          r << Range.new(*(p.scan(/\d+/).map { |x| x.to_i })).map { |x| x.to_i }
+        elsif p =~ /(\*|\d+)\/(\d+)/ && range  # */5, 2/10
+          min = $1 == '*' ? 0 : $1.to_i
+          inc = $2.to_i
+          (min .. range.end).each_with_index do |x, i|
+            r << (range.begin == 1 ? x + 1 : x) if i % inc == 0
+          end
+        else
+          r << p.to_i
+        end
+      end
+      r.flatten
+    end
+  end
+end
+