josip-backgroundrb_merb 1.0.3

data/server/backgroundrb_server.rb

@@ -0,0 +1,7 @@
+require "chronic" 
+require "lib/cron_trigger" 
+require "lib/invalid_dump_error" 
+require "lib/log_worker" 
+require "lib/trigger" 
+require "lib/master_worker" 
+require "lib/meta_worker" 

data/server/lib/cron_trigger.rb

@@ -0,0 +1,195 @@
+module BackgrounDRbMerb
+  class CronTrigger
+    WDAYS = { 0 => "Sunday",1 => "Monday",2 => "Tuesday",3 => "Wednesday", 4 => "Thursday", 5 => "Friday", 6 => "Saturday" }
+    LeapYearMonthDays = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+    CommonYearMonthDays = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
+
+    attr_reader :sec, :min, :hour, :day, :month, :wday, :year, :cron_expr
+
+    def initialize(expr)
+      self.cron_expr = expr
+    end
+
+    def cron_expr=(expr)
+      @cron_expr = expr
+      self.sec, self.min, self.hour, self.day, self.month, self.wday, self.year = @cron_expr.split(' ')
+    end
+    
+    def fire_after_time(p_time)
+      @t_sec,@t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday,@t_yday,@t_idst,@t_zone = p_time.to_a
+      @count = 0
+      loop do 
+        @count += 1
+
+        if @year && !@year.include?(@t_year)
+          return nil if @t_year > @year.max
+          @t_year = @year.detect { |y| y > @t_year }
+        end
+        
+        # if range of months doesn't include current month, find next month from the range
+        unless @month.include?(@t_month)
+          next_month = @month.detect { |m| m > @t_month } || @month.min
+          @t_day,@t_hour,@t_min,@t_sec = @day.min,@hour.min,@min.min,@sec.min
+          if next_month < @t_month 
+            @t_month = next_month
+            @t_year += 1 
+            retry
+          end
+          @t_month = next_month
+        end
+        
+        if !day_restricted? && wday_restricted?
+          unless @wday.include?(@t_wday)
+            next_wday = @wday.detect { |w| w > @t_wday} || @wday.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+            t_time = Chronic.parse("next #{WDAYS[next_wday]}",:now => current_time)
+            @t_day,@t_month,@t_year = t_time.to_a[3..5]
+            @t_wday = next_wday
+            retry
+          end
+        elsif !wday_restricted? && day_restricted?
+          day_range = (1.. month_days(@t_year,@t_month))
+          # day array, that includes days which are present in current month
+          day_array = @day.select { |d| day_range === d }
+          unless day_array.include?(@t_day)
+            next_day = day_array.detect { |d| d > @t_day } || day_array.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+            if !next_day || next_day < @t_day
+              t_time = Chronic.parse("next month",:now => current_time)
+              @t_day = next_day.nil? ? @day.min : next_day
+              @t_month,@t_year = t_time.month,t_time.year
+              retry
+            end
+            @t_day = next_day
+          end
+        else
+          # if both day and wday are restricted cron should give preference to one thats closer to current time
+          day_range = (1 .. month_days(@t_year,@t_month))
+          day_array = @day.select { |d| day_range === d }
+          if !day_array.include?(@t_day) && !@wday.include?(@t_wday)
+            next_day = day_array.detect { |d| d > @t_day } || day_array.min
+            next_wday = @wday.detect { |w| w > @t_wday } || @wday.min
+            @t_hour,@t_min,@t_sec = @hour.min,@min.min,@sec.min
+            
+            # if next_day is nil or less than @t_day it means that it should run in next month
+            if !next_day || next_day < @t_day
+              next_time_mday = Chronic.parse("next month",:now => current_time)
+            else
+              @t_day = next_day
+              next_time_mday = current_time
+            end
+            next_time_wday = Chronic.parse("next #{WDAYS[next_wday]}",:now => current_time)
+            if next_time_mday < next_time_wday
+              @t_day,@t_month,@t_year = next_time_mday.to_a[3..5]
+            else
+              @t_day,@t_month,@t_year = next_time_wday.to_a[3..5]
+            end
+            retry
+          end
+        end
+        
+        unless @hour.include?(@t_hour)
+          next_hour = @hour.detect { |h| h > @t_hour } || @hour.min
+          @t_min,@t_sec = @min.min,@sec.min
+          if next_hour < @t_hour
+            @t_hour = next_hour
+            next_day = Chronic.parse("next day",:now => current_time)
+            @t_day,@t_month,@t_year,@t_wday = next_day.to_a[3..6]
+            retry
+          end
+          @t_hour = next_hour
+        end
+        
+        unless @min.include?(@t_min)
+          next_min = @min.detect { |m| m > @t_min } || @min.min
+          @t_sec = @sec.min
+          if next_min < @t_min
+            @t_min = next_min
+            next_hour = Chronic.parse("next hour",:now => current_time)
+            @t_hour,@t_day,@t_month,@t_year,@t_wday = next_hour.to_a[2..6]
+            retry
+          end
+          @t_min = next_min
+        end
+        
+        unless @sec.include?(@t_sec)
+          next_sec = @sec.detect { |s| s > @t_sec } || @sec.min
+          if next_sec < @t_sec
+            @t_sec = next_sec
+            next_min = Chronic.parse("next minute",:now => current_time)
+            @t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday = next_min.to_a[1..6]
+            retry
+          end
+          @t_sec = next_sec
+        end
+        break
+      end # end of loop do
+      current_time
+    end
+    
+    def current_time
+      Time.local(@t_sec,@t_min,@t_hour,@t_day,@t_month,@t_year,@t_wday,nil,@t_idst,@t_zone)
+    end
+
+    def day_restricted?
+      return !@day.eql?(1..31)
+    end
+
+    def wday_restricted?
+      return !@wday.eql?(0..6)
+    end
+
+    # TODO: mimic attr_reader to define all of these
+    def sec=(sec); @sec = parse_part(sec, 0 .. 59); end
+
+    def min=(min); @min = parse_part(min, 0 .. 59); end
+
+    def hour=(hour); @hour = parse_part(hour, 0 .. 23); end
+
+    def day=(day)
+      @day = parse_part(day, 1 .. 31)
+    end
+
+    def month=(month)
+      @month = parse_part(month, 1 .. 12)
+    end
+
+    def year=(year)
+      @year = parse_part(year)
+    end
+
+    def wday=(wday)
+      @wday = parse_part(wday, 0 .. 6)
+    end
+    private
+    def month_days(y, m)
+      if ((y % 4 == 0) && (y % 100 != 0)) || (y % 400 == 0)
+        LeapYearMonthDays[m-1]
+      else
+        CommonYearMonthDays[m-1]
+      end
+    end
+
+    # 0-5,8,10; 0-5; *; */5
+    def parse_part(part, range=nil)
+      return range  if part.nil? or part == '*' or part =~ /^[*0]\/1$/
+
+      r = Array.new
+      part.split(',').each do |p|
+        if p =~ /-/  # 0-5
+          r << Range.new(*(p.scan(/\d+/).map { |x| x.to_i })).map { |x| x.to_i }
+        elsif p =~ /(\*|\d+)\/(\d+)/ && range  # */5, 2/10
+          min = $1 == '*' ? 0 : $1.to_i
+          inc = $2.to_i
+          (min .. range.end).each_with_index do |x, i|
+            r << (range.begin == 1 ? x + 1 : x) if i % inc == 0
+          end
+        else
+          r << p.to_i
+        end
+      end
+      r.flatten
+    end
+  end
+end
+

data/server/lib/invalid_dump_error.rb

@@ -0,0 +1,4 @@
+module BackgrounDRbMerb
+  class InvalidDumpError < RuntimeError
+  end
+end

data/server/lib/log_worker.rb

@@ -0,0 +1,25 @@
+class LogWorker < Packet::Worker
+  set_worker_name :log_worker
+  attr_accessor :log_file
+
+  def worker_init
+    file = Merb.root / "log" / "backgroundrb_#{CONFIG_FILE[:backgroundrb][:port]}.log"
+    @log_file = Merb::Logger.new(file, :debug, "", true)
+  end
+
+  def receive_data(p_data)
+    case p_data[:type]
+      when :request:  process_request(p_data)
+      when :response: process_response(p_data)
+    end
+  end
+
+  def process_request(p_data)
+    log_data = p_data[:data]
+    @log_file << "[I] " + log_data
+  end
+
+  def process_response
+    puts "Not implemented and needed"
+  end
+end

data/server/lib/master_worker.rb

@@ -0,0 +1,301 @@
+#!/usr/bin/env ruby
+module BackgrounDRbMerb
+  # Class wraps a logger object for debugging internal errors within server
+  class DebugMaster
+    attr_accessor :log_mode,:logger,:log_flag
+    def initialize(log_mode,log_flag = true)
+      @log_mode = log_mode
+      @log_flag = log_flag
+      if @log_mode == :foreground
+        @logger = ::Logger.new(STDOUT)
+      else
+        @logger = ::Logger.new("#{Merb.root}/log/backgroundrb_#{CONFIG_FILE[:backgroundrb][:port]}_debug.log")
+      end
+    end
+
+    def info(data)
+      return unless @log_flag
+      @logger.info(data)
+    end
+
+    def debug(data)
+      return unless @log_flag
+      @logger.debug(data)
+    end
+  end
+
+  class MasterWorker
+    attr_accessor :debug_logger
+    def receive_data p_data
+      debug_logger.info(p_data)
+      @tokenizer.extract(p_data) do |b_data|
+        t_data = Marshal.load(b_data)
+        debug_logger.info(t_data)
+        case t_data[:type]
+        when :do_work: process_work(t_data)
+        when :get_status: process_status(t_data)
+        when :get_result: process_request(t_data)
+        when :start_worker: start_worker_request(t_data)
+        when :delete_worker: delete_drb_worker(t_data)
+        when :all_worker_status: query_all_worker_status(t_data)
+        when :worker_info: pass_worker_info(t_data)
+        when :all_worker_info: all_worker_info(t_data)
+        end
+      end
+    end
+
+    #
+    def pass_worker_info(t_data)
+      worker_name_key = gen_worker_key(t_data[:worker],t_data[:job_key])
+      worker_instance = reactor.live_workers[worker_name_key]
+      info_response = { :worker => t_data[:worker],:job_key => t_data[:job_key]}
+      worker_instance ? (info_response[:status] = :running) : (info_response[:status] = :stopped)
+      send_object(info_response)
+    end
+
+    def all_worker_info(t_data)
+      info_response = []
+      reactor.live_workers.each do |key,value|
+        job_key = (value.worker_key.to_s).gsub(/#{value.worker_name}_?/,"")
+        info_response << { :worker => value.worker_name,:job_key => job_key,:status => :running }
+      end
+      send_object(info_response)
+    end
+
+    def query_all_worker_status(p_data)
+      dumpable_status = { }
+      reactor.live_workers.each { |key,value| dumpable_status[key] = reactor.result_hash[key] }
+      send_object(dumpable_status)
+    end
+
+    # FIXME: although worker key is removed nonetheless from live_workers hash
+    # it could be a good idea to remove it here itself.
+    def delete_drb_worker(t_data)
+      worker_name = t_data[:worker]
+      job_key = t_data[:job_key]
+      worker_name_key = gen_worker_key(worker_name,job_key)
+      begin
+        # ask_worker(worker_name,:job_key => t_data[:job_key],:type => :request, :data => { :worker_method => :exit})
+        worker_instance = reactor.live_workers[worker_name_key]
+        # pgid = Process.getpgid(worker_instance.pid)
+        Process.kill('TERM',worker_instance.pid)
+        # Process.kill('-TERM',pgid)
+
+        # Process.kill('KILL',worker_instance.pid)
+      rescue Packet::DisconnectError => sock_error
+        # reactor.live_workers.delete(worker_name_key)
+        reactor.remove_worker(sock_error)
+      rescue
+        debug_logger.info($!.to_s)
+        debug_logger.info($!.backtrace.join("\n"))
+      end
+    end
+
+    def start_worker_request(p_data)
+      start_worker(p_data)
+    end
+
+    def process_work(t_data)
+      worker_name = t_data[:worker]
+      worker_name_key = gen_worker_key(worker_name,t_data[:job_key])
+      t_data.delete(:worker)
+      t_data.delete(:type)
+      begin
+        ask_worker(worker_name_key,:data => t_data, :type => :request, :result => false)
+      rescue Packet::DisconnectError => sock_error
+        reactor.live_workers.delete(worker_name_key)
+      rescue
+        debug_logger.info($!.to_s)
+        debug_logger.info($!.backtrace.join("\n"))
+        return
+      end
+
+    end
+
+    def process_status(t_data)
+      worker_name = t_data[:worker]
+      job_key = t_data[:job_key]
+      worker_name_key = gen_worker_key(worker_name,job_key)
+      status_data = reactor.result_hash[worker_name_key.to_sym]
+      send_object(status_data)
+    end
+
+    def process_request(t_data)
+      worker_name = t_data[:worker]
+      worker_name_key = gen_worker_key(worker_name,t_data[:job_key])
+      t_data.delete(:worker)
+      t_data.delete(:type)
+      begin
+        ask_worker(worker_name_key,:data => t_data, :type => :request,:result => true)
+      rescue Packet::DisconnectError => sock_error
+        reactor.live_workers.delete(worker_name_key)
+      rescue
+        debug_logger.info($!.to_s)
+        debug_logger.info($!.backtrace.join("\n"))
+        return
+      end
+    end
+
+    # this method can receive one shot status reports or proper results
+    def worker_receive p_data
+      send_object(p_data)
+    end
+
+    def unbind
+      debug_logger.info("Client disconected")
+    end
+    def post_init
+      @tokenizer = BinParser.new
+    end
+    def connection_completed; end
+  end
+
+  class MasterProxy
+    attr_accessor :config_file,:reloadable_workers,:worker_triggers,:reactor
+    def initialize
+      raise "Running old Ruby version, upgrade to Ruby >= 1.8.5" unless check_for_ruby_version
+      @config_file = BackgrounDRbMerb::Config.read_config("#{Merb.root}/config/backgroundrb.yml")
+
+      log_flag = CONFIG_FILE[:backgroundrb][:debug_log].nil? ? true : CONFIG_FILE[:backgroundrb][:debug_log]
+      debug_logger = DebugMaster.new(CONFIG_FILE[:backgroundrb][:log],log_flag)
+
+      load_merb_env
+
+      find_reloadable_worker
+
+      Packet::Reactor.run do |t_reactor|
+        @reactor = t_reactor
+        enable_memcache_result_hash(t_reactor) if CONFIG_FILE[:backgroundrb][:result_storage] && CONFIG_FILE[:backgroundrb][:result_storage][:memcache]
+        t_reactor.start_worker(:worker => :log_worker) if log_flag
+        t_reactor.start_server(CONFIG_FILE[:backgroundrb][:ip],CONFIG_FILE[:backgroundrb][:port],MasterWorker) { |conn|  conn.debug_logger = debug_logger }
+        t_reactor.next_turn { reload_workers }
+      end
+    end
+
+    def gen_worker_key(worker_name,job_key = nil)
+      return worker_name if job_key.nil?
+      return "#{worker_name}_#{job_key}".to_sym
+    end
+
+
+    # method should find reloadable workers and load their schedule from config file
+    def find_reloadable_worker
+      t_workers = Dir["#{WORKER_ROOT}/**/*.rb"]
+      @reloadable_workers = t_workers.map do |x|
+        worker_name = File.basename(x,".rb")
+        require worker_name
+        worker_klass = Object.const_get(worker_name.camel_case)
+        worker_klass.reload_flag ? worker_klass : nil
+      end.compact
+      @worker_triggers = { }
+      @reloadable_workers.each do |t_worker|
+        schedule = load_reloadable_schedule(t_worker)
+        if schedule && !schedule.empty?
+          @worker_triggers[t_worker.worker_name.to_sym] = schedule
+        end
+      end
+    end
+
+    def load_reloadable_schedule(t_worker)
+      worker_method_triggers = { }
+      worker_schedule = CONFIG_FILE[:schedules][t_worker.worker_name.to_sym]
+
+      worker_schedule && worker_schedule.each do |key,value|
+        case value[:trigger_args]
+        when String
+          cron_args = value[:trigger_args] || "0 0 0 0 0"
+          trigger = BackgrounDRbMerb::CronTrigger.new(cron_args)
+        when Hash
+          trigger = BackgrounDRbMerb::Trigger.new(value[:trigger_args])
+        end
+        worker_method_triggers[key] = { :trigger => trigger,:data => value[:data],:runtime => trigger.fire_after_time(Time.now).to_i }
+      end
+      worker_method_triggers
+    end
+
+    # method will reload workers that should be loaded on each schedule
+    def reload_workers
+      return if worker_triggers.empty?
+      worker_triggers.each do |key,value|
+        value.delete_if { |key,value| value[:trigger].respond_to?(:end_time) && value[:trigger].end_time <= Time.now }
+      end
+
+      worker_triggers.each do |worker_name,trigger|
+        trigger.each do |key,value|
+          time_now = Time.now.to_i
+          if value[:runtime] < time_now
+            load_and_invoke(worker_name,key,value)
+            t_time = value[:trigger].fire_after_time(Time.now)
+            value[:runtime] = t_time.to_i
+          end
+        end
+      end
+    end
+
+    # method will load the worker and invoke worker method
+    def load_and_invoke(worker_name,p_method,data)
+      begin
+        require worker_name.to_s
+        job_key = Packet::Guid.hexdigest
+        @reactor.start_worker(:worker => worker_name,:job_key => job_key)
+        worker_name_key = gen_worker_key(worker_name,job_key)
+        data_request = {:data => { :worker_method => p_method,:data => data[:data]},
+          :type => :request, :result => false
+        }
+
+        exit_request = {:data => { :worker_method => :exit},
+          :type => :request, :result => false
+        }
+
+        @reactor.live_workers[worker_name_key].send_request(data_request)
+        @reactor.live_workers[worker_name_key].send_request(exit_request)
+      rescue LoadError
+        puts "no such worker #{worker_name}"
+      rescue MissingSourceFile
+        puts "no such worker #{worker_name}"
+        return
+      end
+    end
+
+    def load_merb_env
+      run_env = CONFIG_FILE[:backgroundrb][:environment] || 'development'
+      lazy_load = CONFIG_FILE[:backgroundrb][:lazy_load].nil? ? true : CONFIG_FILE[:backgroundrb][:lazy_load].nil?
+      require_merb_files unless lazy_load
+      ActiveRecord::Base.allow_concurrency = true if defined?(ActiveRecord)
+    end
+
+    def require_merb_files
+      debug_logger = DebugMaster.new(CONFIG_FILE[:backgroundrb][:log],true)
+
+      files = Dir["#{Merb.root}/app/models/**/*.rb"]
+      files.each { |x|
+        puts "***(M) #{x}"
+        begin
+          require x unless defined? x.camel_case
+        rescue LoadError
+          next
+        rescue MissingSourceFile
+          next
+        end
+      }
+    end
+
+    def enable_memcache_result_hash(t_reactor)
+      require 'memcache'
+      memcache_options = {
+        :c_threshold => 10_000,
+        :compression => true,
+        :debug => false,
+        :namespace => 'backgroundrb_result_hash',
+        :readonly => false,
+        :urlencode => false
+      }
+      cache = MemCache.new(memcache_options)
+      cache.servers = CONFIG_FILE[:backgroundrb][:result_storage][:memcache].split(',')
+      t_reactor.set_result_hash(cache)
+    end
+
+    def check_for_ruby_version; return RUBY_VERSION >= "1.8.5"; end
+
+  end # end of module BackgrounDRb
+end