patriot-workflow-scheduler 0.6.1

data/lib/patriot/tool/patriot_commands/worker_admin.rb

@@ -0,0 +1,60 @@
+module Patriot
+  module Tool
+    module PatriotCommands
+      # remote worker administration tool
+      module WorkerAdmin
+
+        Patriot::Tool::PatriotCommand.class_eval do
+          desc 'worker_admin [options] [start|stop|restart|sleep|wake|status]',
+              'controll remote workers'
+          method_option :all,
+              :aliases  => '-a',
+              :type     => :boolean,
+              :default  => false,
+              :desc     => 'target all worker hosts'
+          method_option :host,
+              :aliases  => '-h',
+              :type     => :string,
+              :desc     => 'target host'
+          def worker_admin(sub_cmd)
+            sub_cmd = "#{sub_cmd}_worker" unless sub_cmd == "status"
+            begin
+              opts = symbolize_options(options)
+              conf        = {:type => 'worker_admin'}
+              conf[:path] = opts[:config] if opts.has_key?(:config)
+              config      = load_config(conf)
+              controller  = Patriot::Controller::WorkerAdminController.new(config)
+              result = controller.send(sub_cmd.to_sym, opts)
+              print_mtd ="print_#{sub_cmd}".to_sym
+              self.send(print_mtd, result, opts) if self.respond_to?(print_mtd)
+            rescue => e
+              puts e
+              raise e
+            end
+          end
+
+          no_commands do 
+            def print_status(result, opts)
+              statuses = {}
+              unless opts[:all]
+                raise "illegal response #{result} from #{opts[:host]}" unless result.size == 1
+                result[opts[:host]] = result.delete(result.keys.first)
+              end
+              result.each do |s,r|
+                if r.nil?
+                  statuses[s] = 'HALT'
+                else
+                  r = JSON.parse(r)
+                  raise "illegal response #{r} from #{s}" unless r.size == 1
+                  statuses[s] = r.values[0].nil? ? 'HALT' : r.values[0]
+                end
+              end
+              puts JSON.generate(statuses)
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/patriot/util.rb

@@ -0,0 +1,14 @@
+module Patriot
+  # namespace for utility functions
+  module Util
+    require 'patriot/util/config'
+    require 'patriot/util/logger'
+    require 'patriot/util/param'
+    require 'patriot/util/date_util'
+    require 'patriot/util/script'
+    require 'patriot/util/retry'
+    require 'patriot/util/db_client'
+    require 'patriot/util/system'
+    require 'patriot/util/cron_format_parser'
+  end
+end

data/lib/patriot/util/config.rb

@@ -0,0 +1,58 @@
+require 'patriot/util/config/base'
+require 'patriot/util/config/inifile_config'
+
+module Patriot
+  module Util
+    # namespace for configuration files
+    module Config
+      # the path to default configuration file
+      DEFAULT_CONFIG = File.join($home || Dir.pwd, 'config', 'patriot.ini')
+      # a configuration parameter key for plugins
+      PLUGIN_KEY = "plugins"
+      # a configuration parameter key for plugins directory
+      PLUGIN_DIR_KEY = "plugins.dir"
+      # default plugins directory
+      DEFAULT_PLUGIN_DIR = "plugins"
+      # plugin directory
+      PLUGIN_LIB_DIR = 'lib' 
+      # plugin initiation script
+      PLUGIN_INIT_SCRIPT = 'init.rb'
+
+      # load configuration file
+      # @param option [Hash]
+      # @option option :path [String] path to configuration file
+      # @option option :type [String] load type (differe by tool)
+      # @option option :ignore_plugin [Boolean] set true not to load plugins
+      def load_config(option = {})
+        option = {:path => DEFAULT_CONFIG, 
+                  :type => nil, 
+                  :ignore_plugin => false }.merge(option)
+        conf = nil
+        case File.extname(option[:path])
+        when '.ini'
+          conf = Patriot::Util::Config::IniFileConfig.new(option[:path], option[:type])
+        else
+          raise "unsupported config file name: #{conf[:path]}"
+        end
+        load_plugins(conf) unless option[:ignore_plugin]
+        return conf
+      end
+
+      # load plugins
+      # @param conf [Patriot::Util::Config::Base] configuration to load plugins
+      def load_plugins(conf)
+        plugins = conf.get(PLUGIN_KEY)
+        return conf if plugins.nil?
+        plugins = [plugins] unless plugins.is_a?(Array)
+        plugin_dir = conf.get(PLUGIN_DIR_KEY, DEFAULT_PLUGIN_DIR)
+        plugins.each do |plugin|
+          path = File.join($home, plugin_dir, plugin)
+          init_script = File.join(path, PLUGIN_INIT_SCRIPT)
+          raise "Failed to load #{plugin}: #{init_script} does not exist" unless File.file?(init_script)
+          $: << File.join(path, PLUGIN_LIB_DIR)
+          require init_script
+        end
+      end
+    end
+  end
+end

data/lib/patriot/util/config/base.rb

@@ -0,0 +1,22 @@
+module Patriot
+  module Util
+    module Config
+      # base class of configuration classes
+      class Base
+        # get value for the specified key
+        # @param [String] key
+        # @param [Object] default default value
+        # @return [Object] the value for the ke
+        def get(key, default=nil)
+          raise NotImplementedError
+        end
+
+        # get path where this configuration is loaded
+        # @return [String] path to the file
+        def path
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/patriot/util/config/inifile_config.rb

@@ -0,0 +1,63 @@
+require 'inifile'
+module Patriot
+  module Util
+    module Config
+      # a configuration implementation definied by the ini-file format
+      class IniFileConfig < Patriot::Util::Config::Base
+
+        # comman section name
+        COMMON_SECTION = 'common'
+
+        # @param path [String] path to a configuration file
+        # @param type [String] load type (section name to be loaded)
+        def initialize(path, type = nil)
+          raise "path in String is expected but #{path.class}" unless path.is_a?(String)
+          @path = path
+          config = IniFile.load(path)
+          raise "#{path} not found" if config.nil?
+          @config = {}
+          read_section(config, COMMON_SECTION)
+          read_section(config, type)
+        end
+
+        # @private
+        # read configuration from a section
+        # @param config [IniFile] ini file configuration
+        # @param section [String] section name
+        def read_section(config, section)
+          sect = config[section]
+          return if sect.nil?
+          sect.each{|k,v| @config[k.to_sym] = v }
+        end
+        private :read_section
+
+        # @see Patriot::Util::Config::Base
+        def path
+          return @path
+        end
+
+        # @see Patriot::Util::Config::Base
+        def get(name, default=nil)
+          v = @config[name.to_sym]
+          v = split_value(v)
+          return v.nil? ? default : v
+        end
+
+        # split configuration value by a delimiter
+        # @param value [String] the value to be splitted
+        # @param delimiter [String] delimiter for splitting
+        def split_value(value, delimiter = ',')
+          # don't allow spaces around value
+          regexp = Regexp.new("\\s*#{delimiter}\\s*")
+          if value.is_a?(String) && value =~ regexp 
+            return value.split(regexp)
+          else
+            return value
+          end
+        end
+        private :split_value
+
+      end
+    end
+  end
+end

data/lib/patriot/util/cron_format_parser.rb

@@ -0,0 +1,104 @@
+require 'date'
+module Patriot
+  module Util
+    # module for handling cron format interval
+    module CronFormatParser
+
+      # start 00:00:00 everyday
+      DEFAULT_CRON_FIELD = "0 0 * * *" 
+
+      # a key word to indicate interval in which jobs are executed on the end of every month
+      END_OF_EVERY_MONTH = 'end_of_every_month'
+
+      # the list of week days
+      WEEKS   = 0.upto(6).to_a
+      # the list of months
+      MONTHS  = 1.upto(12).to_a
+      # the list of days
+      DAYS    = 1.upto(31).to_a
+      # the list of hours
+      HOURS   = 0.upto(23).to_a
+      # the list of minutes
+      MINUTES = 0.upto(59).to_a
+
+      # expand a given date to the array of time which should be executed in case of a given cron field
+      # @param date [DateTime] target datetime
+      # @param cron_field [String] interval in cron format
+      # @return [Array<DateTime>] a list of datetime which match the cron format
+      def expand_on_date(date, cron_field = nil)
+        cron_field = DEFAULT_CRON_FIELD if cron_field.nil? || cron_field.empty?
+        if cron_field == END_OF_EVERY_MONTH
+          return date.next.day == 1 ? [date] : []
+        end
+        field_splits = cron_field.split
+        raise "illegal cron field format #{cron_field}" unless field_splits.size == 5
+        minute, hour, day, month, week = field_splits
+        return [] unless is_target_day?(date, day, month, week)
+        return target_hours(hour).map do |h|
+          target_minutes(minute).map do |m|
+            DateTime.new(date.year, date.month, date.day, h, m, 0)
+          end
+        end.flatten
+      end
+
+      # check a given date is a target or not
+      # @param date [DateTime] a datetime to be checked
+      # @param day [String] day field in cron format
+      # @param month [String] month field in cron format
+      # @param week [String] week field in cron format
+      def is_target_day?(date, day, month, week)
+        unless month == "*" || parse_field(month, MONTHS).to_a.include?(date.month)
+          return false 
+        end
+        return true if day == "*" && week == "*"
+        target_weeks = parse_field(week, WEEKS)
+        return parse_field(day, DAYS).include?(date.day) if week == "*"
+        return parse_field(week, WEEKS).include?(date.wday)
+      end
+
+      # @param hour [String] hour field in cron format
+      # @return [Array<Integer>] a target hours for the given hour specification
+      def target_hours(hour)
+        return parse_field(hour, HOURS)
+      end
+
+      # @param minute [String] minute field in cron format
+      # @return [Array<Integer>] a target minutes for the given minute specification
+      def target_minutes(minute)
+        return parse_field(minute, MINUTES)
+      end
+
+      # select elements which match a given field from a given domain
+      # @param field [String] one of the cron field
+      # @param domain [Array<Integer>] the domain of the field
+      # @return [Array<Integer>] a list of the domain elements match with the field
+      def parse_field(field, domain)
+        field = field.split("/")
+        raise "illegal cron field format #{field.join("/")}" unless field.size <= 2 
+        range = []
+        if field[0] == "*"
+          range = domain 
+        else
+          range = field[0].split(",").map do |r|
+            subrange = r.split("-")
+            if subrange.size == 1
+              subrange[0].to_i
+            elsif subrange.size == 2 
+              subrange[0].to_i.upto(subrange[1].to_i).to_a
+            else
+              raise "illegal cron field format #{field.join("/")}" 
+            end
+          end.flatten
+        end
+        return range if field.size == 1
+        interval = field[1].to_i
+        filtered_range = []
+        range.each_with_index do |r,i| 
+          filtered_range << r if (i % interval) == 0
+        end
+        return filtered_range
+      end
+
+    end
+  end
+end

data/lib/patriot/util/date_util.rb

@@ -0,0 +1,200 @@
+require 'date'
+module Patriot
+  module Util
+    # namesapce for date functions
+    module DateUtil
+
+      # format date
+      # @param dt [String] date in '%Y-%m-%d'
+      # @param time [String] time in '%H:%M:%S'
+      def date_format(dt, time, fmt, diff={})
+        diff = {:year => 0, 
+                :month => 0,
+                :day => 0,
+                :hour => 0,
+                :min => 0,
+                :sec => 0}.merge(diff)
+
+        dt  = eval "\"#{dt}\""
+        time  = eval "\"#{time}\""
+
+        t = time.split(':')
+        d = dt.split('-')
+        sec = t[2].to_i + diff[:sec]
+        min = t[1].to_i + diff[:min] 
+        hour = t[0].to_i + diff[:hour] 
+        day = d[2].to_i 
+        month = d[1].to_i 
+        year = d[0].to_i + diff[:year]
+
+        min = min+ sec/60
+        hour = hour + min/60
+        diff[:day]= diff[:day] + hour/24  
+
+        sec = sec%60
+        min = min%60
+        hour = hour%24
+   
+        new_dt = DateTime.new(year, month, day, hour, min,sec)
+        new_dt = new_dt >> diff[:month]
+        new_dt = new_dt + diff[:day]
+        return new_dt.strftime(fmt)
+      end
+
+      # add interval to date
+      # @param date [String] date in '%Y-%m-%d'
+      # @param interval [Integer] interval in day
+      # @return [String] date in '%Y-%m-%d'
+      def date_add(date, interval)
+        d = to_date_obj(date)
+        d = d + interval
+        return d.strftime('%Y-%m-%d')
+      end
+
+      # subtract interval from date
+      # @param date [String] date in '%Y-%m-%d'
+      # @param interval [Integer] interval in day
+      # @return [String] date in '%Y-%m-%d'
+      def date_sub(date, interval)
+        d = to_date_obj(date)
+        d = d - interval
+        return d.strftime('%Y-%m-%d')
+      end
+
+      # @param date [String] date in '%Y-%m-%d'
+      # @param interval [Integer] interval in year
+      # @return [String] date of some years later in '%Y-%m-%d'
+      def date_add_year(date, interval)
+        s = date.split("-");
+        return Date.new(s[0].to_i + interval,s[1].to_i,s[2].to_i).strftime("%Y-%m-%d")
+      end
+
+      # @param date [String] date in '%Y-%m-%d'
+      # @param interval [Integer] interval in year
+      # @return [String] date of some years before in '%Y-%m-%d'
+      def date_sub_year(date, interval)
+        s = date.split("-");
+        return Date.new(s[0].to_i - interval,s[1].to_i,s[2].to_i).strftime("%Y-%m-%d")
+      end
+
+      # add interval to month
+      # @param month [String] month in '%Y-%m'
+      # @param interval [Integer] interval in month
+      # @return [String] month in '%Y-%m'
+      def month_add(month, interval)
+        d = to_date_obj("#{month}-01")
+        d = d >> interval
+        return d.strftime('%Y-%m')
+      end
+
+      # subtract interval from month
+      # @param month [String] month in '%Y-%m'
+      # @param interval [Integer] interval in month
+      # @return [String] month in '%Y-%m'
+      def month_sub(month, interval)
+        d = to_date_obj("#{month}-01")
+        d = d << interval
+        return d.strftime('%Y-%m')
+      end
+
+      # convert to month expression
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [String] month in '%Y-%m'
+      def to_month(date)
+        d = to_date_obj(date)
+        return d.strftime('%Y-%m')
+      end
+
+      # get first date of the month
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [String] date in '%Y-%m-%d'
+      def to_start_of_month(date)
+        s = date.split("-");
+        return Date.new(s[0].to_i,s[1].to_i,1).strftime("%Y-%m-%d")
+      end
+
+      # get the last date of the month
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [String] date in '%Y-%m-%d'
+      def to_end_of_month(date)
+        s = date.split("-");
+        return ((Date.new(s[0].to_i,s[1].to_i,1)>>1)-1).strftime("%Y-%m-%d")
+      end
+
+      # get the last date of the last month
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [String] date in '%Y-%m-%d'
+      def to_end_of_last_month(date)
+        d = to_date_obj(date)
+        d = d - d.day
+        return d.strftime('%Y-%m-%d')
+      end
+
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [Integer] week of the date in (0-6)
+      def days_of_week(date)
+        d = to_date_obj(date)
+        return d.wday
+      end
+
+      # @param month [String] month in '%Y-%m'
+      # @return [Array<String>] the list of days in the month
+      def days_of_month(month)
+        s = month.split('-')
+        y = s[0].to_i
+        m = s[1].to_i
+        return 1.upto(31).map{|d| Date.new(y,m,d).strftime('%Y-%m-%d') if Date.valid_date?(y,m,d)}.compact
+      end
+
+      # @param dt [String] date in '%Y-%m-%d'
+      # @return [Array<String>] the list of days between the first date and the given date in the month
+      def days_of_month_until(dt)
+        s = dt.split('-')
+        y = s[0].to_i
+        m = s[1].to_i
+        d = s[2].to_i
+        return 1.upto(d).map{|d| Date.new(y,m,d).strftime('%Y-%m-%d') if Date.valid_date?(y,m,d)}.compact
+      end
+
+      # convert to a Date instance
+      # @param date [String] date in '%Y-%m-%d'
+      # @return [Date]
+      def to_date_obj(date)
+        d = date.split('-')
+        return Date.new(d[0].to_i, d[1].to_i, d[2].to_i)
+      end
+
+      # @deprecated
+      def date_to_month(date)
+        d = date.split('-')
+        return Date.new(d[0].to_i, d[1].to_i, d[2].to_i).strftime('%Y-%m')
+      end
+
+      # @return [Array<String>] a list of hours (00..23)
+      def hours
+        return 0.upto(23).map do |h| h_str = h.to_s.rjust(2, "0") end.flatten
+      end
+
+      # validate date or date range expression
+      def validate_and_parse_dates(date)
+        date_objs = date.split(",").map do |d|
+          unless d.to_s =~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/
+            raise ArgumentError, "ERROR: invalid date #{d} in #{date}"
+          end
+          Date.parse(d.to_s)
+        end
+        if date_objs.size == 1
+          return [date]
+        elsif date_objs.size == 2
+          dates = []
+          date_objs[0].upto(date_objs[1]){|d| dates << d.to_s}
+          return dates
+        end
+        raise ArgumentError, "ERROR: invalid date #{date}"
+      end
+      private :validate_and_parse_dates
+
+    end
+  end
+end
+