patriot-workflow-scheduler 0.6.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MjRlNzEwZGJjMmY4MTRjMDU1NjBhYTg5OTRlNWYzYmUyZjQ5MWE3MQ==
+  data.tar.gz: !binary |-
+    ZTA3YjNiN2JjMDczMjZkYWEzNTc3NWJhOWZjYmNkOTdhZTU1MzM5ZQ==
+SHA512:
+  metadata.gz: !binary |-
+    ODY5NzRjNWZjYmU2MzU4N2E4M2U5YThlODRkN2I2NThlZjI3ZTUxYzcwZTg0
+    MDJmOGZhZTE5ZTE4ZWYzNWIwYzkyZmRhYTkwOTBmZWYxYWE0Mjg3YmQ1MWZl
+    ODgyMTljODVjNGU0ZmE5OTJlZGM5ODkxMzQ4MzIxY2I4ZmZjMGY=
+  data.tar.gz: !binary |-
+    YzMzYmU3YzU3ZGQ5YWUxNGRmNTYyNmM0NTMwMzI4OWVhNDgyODFiNDZkZTI5
+    NmExNTNjOGI0NmY3YTNiYzE1MTk4N2NhYTYxNGExNDQyNWM5NzhmNDA3NmU1
+    ZTM1NzIwZmNhMDZlNGIwZmE5NjdkNzczMmM3MzI4Njg4NDcyMDY=

data/bin/patriot

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+Encoding.default_external = 'utf-8'
+$home = File.expand_path(File.dirname(File.expand_path(__FILE__)) + "/..")
+$: << File.join($home,"lib")
+require 'rubygems'
+require 'patriot'
+
+Patriot::Tool::PatriotCommand.start(ARGV)

data/bin/patriot-init

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+
+target=ARGV[0]
+unless File.exist?(target)
+  puts "directory #{target} does not exist"
+  Dir.mkdir(target)
+end
+base_dir = File.join(File.dirname(File.expand_path(__FILE__)), "..")
+skel_dir = File.join(base_dir, "skel")
+Dir.glob(File.join(skel_dir,"*")).each do |d|
+  if File.exist?(File.join(target, File.basename(d)))
+    puts "#{d} already exists"
+  else
+    puts "copy #{d} to  #{target}"
+    FileUtils.cp_r(d, target)
+  end
+end
+
+
+def mkdir_ifnot_exist(dir)
+  if File.exist?(dir)
+    puts "#{dir} already exist"
+  else
+    puts "mkdir #{dir}"
+    FileUtils.mkdir_p(dir)
+  end
+end
+
+mkdir_ifnot_exist(File.join(target, "bin"))
+FileUtils.cp(File.join(base_dir,"bin","patriot"),File.join(target,"bin","patriot"))
+mkdir_ifnot_exist(File.join(target, "plugins"))
+mkdir_ifnot_exist(File.join(target, "run"))
+
+

data/lib/patriot.rb

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+require 'rubygems'
+require 'active_support'
+
+require 'patriot/util'
+require 'patriot/command'
+require 'patriot/job_store'
+require 'patriot/worker'
+require 'patriot/controller'
+require 'patriot/tool'
+

data/lib/patriot/command.rb

@@ -0,0 +1,71 @@
+# the root name space for this scheduler
+module Patriot
+  # a name space for commands
+  module Command 
+    # a parameter key for command class name (used in jobs)
+    COMMAND_CLASS_KEY   = "COMMAND_CLASS"
+
+    # attribute name for required products
+    REQUISITES_ATTR     = "requisites"
+    # attribute name for produced products
+    PRODUCTS_ATTR       = "products"
+    # attribute name for job state
+    STATE_ATTR          = "state"
+    # attribute name for job priority
+    PRIORITY_ATTR       = "priority"
+    # attribute name for job execution node constraint
+    EXEC_NODE_ATTR      = "exec_node"
+    # attribute name for job execution host constraint
+    EXEC_HOST_ATTR      = "exec_host"
+    # attribute name for start time constraint
+    START_DATETIME_ATTR = "start_datetime"
+    # attribute name for the skip on fail marker
+    SKIP_ON_FAIL_ATTR   = "skip_on_fail"
+
+    # a list of comman attributes
+    COMMON_ATTRIBUTES = [
+      REQUISITES_ATTR,
+      PRODUCTS_ATTR,
+      STATE_ATTR,
+      PRIORITY_ATTR,
+      EXEC_NODE_ATTR,
+      EXEC_HOST_ATTR,
+      START_DATETIME_ATTR,
+      SKIP_ON_FAIL_ATTR
+    ]
+
+    # exit code of a command
+    module ExitCode
+      # successfully finished
+      SUCCEEDED    = 0
+      # failed
+      FAILED       = 1
+      # failed but skipped (marked skip_on_fail)
+      FAILURE_SKIPPED = 2
+      # skip (e.g., updated elsewhere)
+      SKIPPED      = -1
+
+      # @param exit_code [Patriot::Command::ExitCode]
+      # @return [String] string expression of the exit code
+      def name_of(exit_code)
+        exit_code = exit_code.to_i
+        return case exit_code
+          when 0  then "SUCCEEDED"
+          when 1  then "FAILED"
+          when 2  then "FAILURE_SKIPPED"
+          when 4  then "FAILED" # for backward compatibility
+          else raise "unknown exit_code #{exit_code}"
+        end
+      end
+      module_function :name_of
+    end
+
+    require 'patriot/command/parser'
+    require 'patriot/command/command_macro'
+    require 'patriot/command/base'
+    require 'patriot/command/command_group'
+    require 'patriot/command/composite'
+    require 'patriot/command/sh_command'
+  end
+end
+

data/lib/patriot/command/base.rb

@@ -0,0 +1,199 @@
+require 'active_support/core_ext/hash/deep_merge'
+module Patriot
+  module Command
+    # The base class of every command.
+    # The command is an executable form of the job.
+    class Base
+      include Patriot::Command::Parser
+      include Patriot::Util::Param
+      include Patriot::Util::DateUtil
+      include Patriot::Util::Logger
+
+      class << self
+        include Patriot::Command::CommandMacro
+      end
+
+      attr_accessor :parser, :test_mode, :target_datetime
+
+      # comman attributes handled distinctively (only effective in top level commands)
+      volatile_attr :requisites, :products, :priority, :start_after, :exec_date, :exec_node, :exec_host, :skip_on_fail
+
+      # @param config [Patriot::Util::Config::Base] configuration for this command
+      def initialize(config)
+        @config     = config
+        @logger     = create_logger(config)
+        @param      = {}
+        @requisites = []
+        @products   = []
+        @macros     = {}
+        @test_mode  = false
+      end
+
+      # convert this to a job so that it can be stored to JobStore
+      # @return [Patriot::JobStore::Job] a job for this command.
+      def to_job
+        job = Patriot::JobStore::Job.new(self.job_id)
+        job.read_command(self)
+        return job
+      end
+
+      # get the value of an attribute
+      # @param attr_name [String] attribute name
+      # @return [Object] the value of the attribute specified with argument
+      def [](attr_name)
+        return instance_variable_get("@#{attr_name}".to_sym)
+      end
+
+      # build the identifier of the job for this command.
+      # This method should be overriden in sub-classes
+      # @return [String] the identifier of the job.
+      def job_id
+        raise NotImplementedError
+      end
+
+      # set default command name
+      # replace :: to handle in JSON format
+      # @return [String] a simplified command namd
+      def command_name
+        return self.class.to_s.split("::").last.downcase.gsub(/command/,"")
+      end
+
+      # add products required by this job.
+      # @param requisites [Array<String>] a list of products required by this job.
+      def require(requisites)
+        return if requisites.nil?
+        @requisites |= requisites.flatten
+      end
+
+      # add products produced by this job.
+      # @param products [Array<String>] a list of products produced by this job
+      def produce(products)
+        return if products.nil?
+        @products |= products.flatten
+      end
+
+      # mark this job to skip execution
+      def skip
+        param 'state' => Patriot::JobStore::JobState::SUCCEEDED
+      end
+
+      # mark this job to suspend execution
+      def suspend
+        param 'state' => Patriot::JobStore::JobState::SUSPEND
+      end
+
+      # mark this job to skip in case of failures
+      def skip_on_fail?
+        return @skip_on_fail == 'true' || @skip_on_fail == true
+      end
+
+      # @return [String] the target month in '%Y-%m'
+      def _month_
+        return @target_datetime.strftime("%Y-%m")
+      end
+
+      # @return [String] the target date in '%Y-%m-%d'
+      def _date_
+        return @target_datetime.strftime("%Y-%m-%d")
+      end
+
+      # @return [Integer] the tergat hour
+      def _hour_
+        return @target_datetime.hour
+      end
+
+      # start datetime of this command.
+      # This command should be executed after the return value of this method
+      # @return [DateTime]
+      def start_date_time
+        return nil if @exec_date.nil? && @start_after.nil?
+        # set tomorrow as default
+        date = (@exec_date || date_add(_date_, 1)).split("-").map(&:to_i)
+        # set midnight as default
+        time = (@start_after || "00:00:00").split(":").map(&:to_i)
+        return DateTime.new(date[0], date[1], date[2], time[0], time[1], time[2])
+      end
+
+      # update parameters with a given hash.
+      # If the hash includes keys which values have already been defined,
+      # the value for the key is replaced with the new value.
+      # @param _param [Hash] a hash from attribute name to its value
+      def param(_param)
+        @param = @param.deep_merge(_param)
+      end
+
+      # build this command as executables
+      # @param _param [Hash] default parameter
+      def build(_param={})
+        @param = _param.deep_merge(@param)
+        init_param
+        @start_datetime = start_date_time
+        cmds = configure()
+        cmds = [cmds] unless cmds.is_a?(Array)
+        cmds.each(&:validate_command_attrs)
+        return cmds.flatten
+      end
+
+      # initialize command attributes
+      def init_param
+        # set parameter value to instance variable
+        @param.each do |k,v|
+          raise "a reserved word #{k} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == k
+          raise "#{k} is already used in #{self.job_id}" unless instance_variable_get("@#{k}".to_sym).nil?
+          # don't evaluate here since all parameters are not set to instance variables
+          instance_variable_set("@#{k}".to_sym,v)
+        end
+
+        # evaluate command attributes using parameters (instance variables)
+        self.class.command_attrs.each { |a, d| configure_attr(a, d) }
+      end
+      protected :init_param
+
+      # configure a command attribute and set as an instance variable
+      # @param attr [String] an attribute name to be configured
+      # @param default_value default value of the attribute
+      def configure_attr(attr, default_value = nil)
+        v = instance_variable_get("@#{attr}".to_sym)
+        v = default_value if v.nil?
+        instance_variable_set("@#{attr}".to_sym, eval_attr(v))
+      end
+
+      # a hook method to implement comand-specific configuration
+      def configure
+        return self
+      end
+
+      # validate values of command attributes
+      # @see Patriot::Command::CommandMacro#validate_attr
+      def validate_command_attrs
+        self.class.validation_logics.each do |attr, logics|
+          val = self.instance_variable_get("@#{attr}".to_sym)
+          logics.each do |l|
+            unless l.call(self, attr, val)
+              raise "validation error : #{attr}=#{val} (#{self.class})" 
+            end
+          end
+        end
+      end
+
+      # add a sub command
+      # @raise if sub command is not supported
+      def add_subcommand
+        raise "sub command is not supported"
+      end
+
+      # @return description of this command
+      def description
+        self.job_id
+      end
+
+      # execute this command
+      def execute()
+        raise NotImplementedError
+      end
+
+    end
+  end
+end
+
+

data/lib/patriot/command/command_group.rb

@@ -0,0 +1,43 @@
+module Patriot
+  module Command 
+    # define a group of jobs
+    class CommandGroup < Base
+      declare_command_name :command_group
+      declare_command_name :job_group
+      attr_accessor :subcommands
+
+      # @see Patriot::Command::Base#initialize
+      def initialize(config)
+        super
+        @subcommands = []
+      end
+
+      # add a command to this group
+      # @param cmd [Patriot::Command::Base] a command to be added to this group
+      def add_subcommand(cmd)
+        @subcommands << cmd
+      end
+
+      # configure thie group.
+      # pass the required/produced products and parameters to the commands in this group
+      # @see Patriot::Command::Base#configure
+      # @return [Array<Patriot::Command::Base>] a list of commands in this group
+      def configure
+        return @subcommands.map{|cmd|
+          cmd.require @requisites
+          cmd.produce @products
+          cmd.build(@param)
+        }.flatten
+      end
+
+      # execute each command in this group
+      # @see Patriot::Command::Base#execute
+      def execute
+        @subcommands.each do |k,v|
+          v.execute
+        end
+      end
+
+    end
+  end
+end

data/lib/patriot/command/command_macro.rb

@@ -0,0 +1,141 @@
+module Patriot
+  module Command
+    # macros to be used for implmentation of command classes
+    module CommandMacro
+
+      # add module methods to DSL methods
+      # @param command_module [Module] module defines methods for the use in DSL
+      def add_dsl_function(command_module)
+        class_eval{ include command_module } 
+      end
+
+      # declare DSL method name for defining the command
+      # @param mth_name [String] the DSL method name
+      # @param parent_cls [Class] parent command to which the new command is added
+      # @param cmd_cls [Class]
+      def declare_command_name(mth_name, parent_cls=Patriot::Command::CommandGroup, cmd_cls=self)
+        parent_cls.class_eval do
+          define_method(mth_name) do |&cmd_def|
+            cmd = new_command(cmd_cls, &cmd_def)
+            add_subcommand(cmd)
+          end
+        end
+      end
+
+      # declare command attributes to be able to use in DSL
+      # @param attrs [String|Hash] attribute names or a Hash from attribute name to its default value
+      # @yield block to preprocess attribute values
+      # @yieldparam [Patriot::Command::Base] command instance
+      # @yieldparam [String] attribute name
+      # @yieldparam [String] passed value
+      def command_attr(*attrs, &blk)
+        @command_attrs = {} if @command_attrs.nil?
+        default_values = {} 
+        if attrs.size == 1 && attrs[0].is_a?(Hash)
+          default_values = attrs[0] 
+          attrs = default_values.keys
+        end
+        attrs.each do |a|
+          raise "a reserved word #{a} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == a
+          raise "#{a} is already defined" if self.instance_methods.include?(a)
+          @command_attrs[a] = default_values[a] 
+          define_method(a) do |*args|
+            raise "illegal size of arguments (#{a} with #{args.inspect})" unless args.size == 1
+            val = args[0]
+            if block_given?
+              yield(self, a, val) 
+            else
+              self.param a.to_s => val
+            end
+          end
+          attr_writer a
+        end
+        return attrs
+      end
+
+      # declare command attributes for only internal use
+      # @param attrs [String] attribute name
+      def private_command_attr(*attrs)
+        command_attr(*attrs) do |cmd, attr_name, attr_val|
+          raise "only internal call is expected for #{attr_name}"
+        end
+      end
+
+      # declare command attributes to be able to use in DSL
+      # values of thes attributes are supposed to be used in {Patriot::Command::Base#do_configure}
+      # and would not be serialized
+      # @param attrs [String|Hash] attribute names or a Hash from attribute name to its default value
+      # @yield block to preprocess attribute values
+      # @yieldparam [Patriot::Command::Base] command instance
+      # @yieldparam [String] attribute name
+      # @yieldparam [String] passed value
+      def volatile_attr(*attrs, &blk)
+        @volatile_attrs = [] if @volatile_attrs.nil?
+        attrs = command_attr(*attrs, &blk)
+        attrs.each do |a|
+          @volatile_attrs << a
+        end
+      end
+
+      # @return [Hash] a Hash from attribute name to its value
+      def command_attrs
+        super_attrs = {}
+        if self.superclass.respond_to?(:command_attrs)
+          super_attrs = self.superclass.command_attrs
+        end
+        return super_attrs if @command_attrs.nil?
+        return super_attrs.merge(@command_attrs)
+      end
+
+      # @return [Array] a list of volatile attribute names
+      def volatile_attrs
+        super_attrs = []
+        if self.superclass.respond_to?(:volatile_attrs)
+          super_attrs = self.superclass.volatile_attrs 
+        end
+        return super_attrs if @volatile_attrs.nil?
+        return @volatile_attrs | super_attrs
+      end
+
+      # @return [Array] a array of attribute names which should be inclued in Job
+      def serde_attrs
+        return command_attrs.keys - volatile_attrs
+      end
+
+      # validate attriubte value
+      # @param attrs attribute names to be validated
+      # @yield logic which validates the value
+      # @yieldreturn [Boolean] true if the value is valid, otherwise false
+      def validate_attr(*attrs, &blk)
+        raise "validation logic not given" unless block_given?
+        unless blk.arity == 3
+          raise "validation logic arguments should be (cmd, attr_name, attr_val)" 
+        end
+        @validation_logics = {} if @validation_logics.nil?
+        attrs.each do |a|
+          raise "#{a} is not command attr" unless command_attrs.include?(a)
+          @validation_logics[a] = [] unless @validation_logics.has_key?(a)
+          @validation_logics[a] << blk
+        end
+      end
+
+      # check whether the attribute value is not nil
+      # @see {validate_attribute_value}
+      def validate_existence(*attrs)
+        validate_attr(*attrs) do |cmd, a, v|
+          !v.nil?
+        end
+      end
+
+      # @return [Hash] a hash from attribute name to validation logic (Process)
+      def validation_logics
+        super_logics = {}
+        if self.superclass.respond_to?(:volatile_attrs)
+          super_logics = self.superclass.validation_logics 
+        end
+        return super_logics if @validation_logics.nil?
+        return super_logics.merge(@validation_logics)
+      end
+    end
+  end
+end