patriot-workflow-scheduler 0.6.2 → 0.7.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OWI3MzFlYjlmYmEzYjBlNTdhNmEzZWJiMDQxMDFkN2FkMjQ3MzFlMA==
+    MjBkMGRiMGQ3ZGU1MTk0OThiN2VkNjBhYmU3YjA1YjVjMmQ4OGU1Mg==
   data.tar.gz: !binary |-
-    NWI0ZDM4ZDVlNzg3OWM5MmI1ZWVhYjkzMWE4MDE0ZDBmNzRhMTkwOQ==
+    ODMwYTA4NWZhNzQ0MWY1M2ZkZDg0NjI4ZmIwYTA3ZTkzOGRlZTQyZg==
 SHA512:
   metadata.gz: !binary |-
-    NTRmYWYyOWMzNzJkZGRjOWY0N2VjYzhiNzEyYTkzNjMzYTBmMTA4OWEzNjM5
-    Y2ZiNGI3MTQ3NDcyNTdmNjEwN2VmOGE1Mzk5NWQ3NjM1ZDEyYTJkNmYzNzU3
-    NzMzNjc4ZjkyOTJjNzU1YjY0NTJhMGZjOTg2MDYyNDA5ZDE1M2Y=
+    YzRkMTVjOThmYTYzYTIxODEwMmNhODNjZGQ1YTBmYzBjYzVjNDhjODg0OTdj
+    NDBkZDZhOTkwNDFhYjM0ZjZlYjlkNzQ4ZWUzMDc1NmU4MWJjMWNkYzc5ZGE1
+    ZWIxNjU1MDJhYTE0MWI1YmQ2YTE3NjE4OTQ0NGFhNzNkZjk4MGU=
   data.tar.gz: !binary |-
-    MDBjZjM1YzZlYmQ4NmJjNWM4NDg1ZWFlMTE4ZjQxNDNmOWQ0OGEyNGJjODQ0
-    NzNiMWYwZDU2NTE3Njc3NzU1ZjE3Y2YzY2IzMTkxZTlhODIyNjA1YWQ3Mzk3
-    NWVlMjk5YjM3MzRmZTVjNTMwNWFlNjY1MzRmOTQyZGQzNDM0ZDY=
+    ODdkZTM5YjliNWI5MDI3YmNkNzVlM2FkNzNkNzg4MDI5YTM0ZDYzNzJkZTc3
+    ODlkMDljNGE1YzA0MWVhNWM4MzAyOTExNDMzNGY1YWEyOGU5YzdhM2M1YTc1
+    M2UxYzMzMzM0NDM1ZjdjNWZkZGM4MmRkYzAxNTc0NTRkMmJlYTg=

data/lib/patriot/command.rb

@@ -19,8 +19,8 @@ module Patriot
     EXEC_HOST_ATTR      = "exec_host"
     # attribute name for start time constraint
     START_DATETIME_ATTR = "start_datetime"
-    # attribute name for the skip on fail marker
-    SKIP_ON_FAIL_ATTR   = "skip_on_fail"
+    # attribute name for retry configration
+    POST_PROCESSORS_ATTR = "post_processors"
 
     # a list of comman attributes
     COMMON_ATTRIBUTES = [
@@ -31,7 +31,7 @@ module Patriot
       EXEC_NODE_ATTR,
       EXEC_HOST_ATTR,
       START_DATETIME_ATTR,
-      SKIP_ON_FAIL_ATTR
+      POST_PROCESSORS_ATTR
     ]
 
     # exit code of a command
@@ -40,8 +40,6 @@ module Patriot
       SUCCEEDED    = 0
       # failed
       FAILED       = 1
-      # failed but skipped (marked skip_on_fail)
-      FAILURE_SKIPPED = 2
       # skip (e.g., updated elsewhere)
       SKIPPED      = -1
 
@@ -50,19 +48,30 @@ module Patriot
       def name_of(exit_code)
         exit_code = exit_code.to_i
         return case exit_code
-          when 0  then "SUCCEEDED"
-          when 1  then "FAILED"
-          when 2  then "FAILURE_SKIPPED"
-          when 4  then "FAILED" # for backward compatibility
-          else raise "unknown exit_code #{exit_code}"
+          when SUCCEEDED  then "SUCCEEDED"
+          when FAILED     then "FAILED"
+          else exit_code.to_s # not nil for backward compatibility
         end
       end
       module_function :name_of
+
+      # @param code_name [Patriot::Command::ExitCode]
+      # @return [Fixnum] exit code of the code name
+      def value_of(code_name)
+        return code_name if code_name.is_a?(Fixnum)
+        return case code_name
+        when /SUCCEEDED/i then SUCCEEDED
+        when /FAILED/i    then FAILED
+        else raise "unknown exit code name: #{code_name}"
+        end
+      end
+      module_function :value_of
     end
 
     require 'patriot/command/parser'
     require 'patriot/command/command_macro'
     require 'patriot/command/base'
+    require 'patriot/command/post_processor'
     require 'patriot/command/command_group'
     require 'patriot/command/composite'
     require 'patriot/command/sh_command'

data/lib/patriot/command/base.rb

@@ -13,10 +13,17 @@ module Patriot
         include Patriot::Command::CommandMacro
       end
 
-      attr_accessor :parser, :test_mode, :target_datetime
+      attr_accessor :config, :parser, :test_mode, :target_datetime, :post_processors
+      attr_writer   :start_datetime
 
       # comman attributes handled distinctively (only effective in top level commands)
-      volatile_attr :requisites, :products, :priority, :start_after, :exec_date, :exec_node, :exec_host, :skip_on_fail
+      volatile_attr :requisites,
+                    :products,
+                    :priority,
+                    :start_after,
+                    :exec_date,
+                    :exec_node,
+                    :exec_host
 
       # @param config [Patriot::Util::Config::Base] configuration for this command
       def initialize(config)
@@ -82,11 +89,6 @@ module Patriot
         param 'state' => Patriot::JobStore::JobState::SUSPEND
       end
 
-      # mark this job to skip in case of failures
-      def skip_on_fail?
-        return @skip_on_fail == 'true' || @skip_on_fail == true
-      end
-
       # @return [String] the target month in '%Y-%m'
       def _month_
         return @target_datetime.strftime("%Y-%m")
@@ -109,14 +111,14 @@ module Patriot
 
       # start datetime of this command.
       # This command should be executed after the return value of this method
-      # @return [DateTime]
+      # @return [Time]
       def start_date_time
         return nil if @exec_date.nil? && @start_after.nil?
         # set tomorrow as default
         date = (@exec_date || date_add(_date_, 1)).split("-").map(&:to_i)
         # set midnight as default
         time = (@start_after || "00:00:00").split(":").map(&:to_i)
-        return DateTime.new(date[0], date[1], date[2], time[0], time[1], time[2])
+        return Time.new(date[0], date[1], date[2], time[0], time[1], time[2])
       end
 
       # update parameters with a given hash.
@@ -187,6 +189,13 @@ module Patriot
         raise "sub command is not supported"
       end
 
+      # add a post processor
+      # @param [Patriot::Command::PostProcessor::Base] a post processor for this job
+      def add_post_processor(post_processor)
+        @post_processors ||= []
+        @post_processors << post_processor
+      end
+
       # @return description of this command
       def description
         self.job_id

data/lib/patriot/command/command_group.rb

@@ -26,6 +26,7 @@ module Patriot
         return @subcommands.map{|cmd|
           cmd.require @requisites
           cmd.produce @products
+          cmd.post_processors = @post_processors + (cmd.post_processors || []) unless @post_processors.nil?
           cmd.build(@param)
         }.flatten
       end

data/lib/patriot/command/parser.rb

@@ -18,7 +18,7 @@ module Patriot
       end
 
       # parse DSL by processing the DSL description as block
-      # @param datetime [DateTime] the datetime for which the job works
+      # @param datetime [Time] the datetime for which the job works
       # @return a list of command defined in the DSL description
       def parse(datetime, blk)
         self.target_datetime = datetime

data/lib/patriot/command/post_processor.rb

@@ -0,0 +1,14 @@
+# the root name space for this scheduler
+module Patriot
+  # a name space for commands
+  module Command 
+    module PostProcessor
+      POST_PROCESSOR_CLASS_KEY   = "POST_PROCESSOR_CLASS"
+      require 'patriot/command/post_processor/base'
+      require 'patriot/command/post_processor/skip_on_fail'
+      require 'patriot/command/post_processor/retrial'
+      require 'patriot/command/post_processor/mail_notification'
+    end
+  end
+end
+

data/lib/patriot/command/post_processor/base.rb

@@ -0,0 +1,50 @@
+module Patriot
+  module Command
+    module PostProcessor
+
+      # The base class of every post processor
+      class Base
+
+        # declare DSL method name for adding a post processor
+        # @param mth_name [String] the DSL method name
+        # @param parent_cls [Class<Patriot::Command::Base>] parent command in which the post porcessor is available
+        # @param processor_cls [Class<Patriot::Command::PostProcessor::Base] the class of the post processor
+        def self.declare_post_processor_name(mth_name, parent_cls=Patriot::Command::Base, processor_cls=self)
+          parent_cls.class_eval do
+            define_method(mth_name) do |processor_props = {}|
+              pp = processor_cls.new(processor_props)
+              add_post_processor(pp)
+            end
+          end
+        end
+
+        attr_accessor :props
+
+        # @param props [Hash] properties of this post processor
+        def initialize(props = {})
+          validate_props(props)
+          @props = props
+        end
+
+        def validate_props(props)
+        end
+
+        def process(cmd, worker, job_ticket)
+          case job_ticket.exit_code
+          when Patriot::Command::ExitCode::SUCCEEDED then process_success(cmd, worker, job_ticket)
+          when Patriot::Command::ExitCode::FAILED    then process_failure(cmd, worker, job_ticket)
+          end
+        end
+
+        def process_success(cmd, worker, job_ticket)
+        end
+
+        def process_failure(cmd, worker, job_ticket)
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/patriot/command/post_processor/mail_notification.rb

@@ -0,0 +1,59 @@
+require 'mail'
+module Patriot
+  module Command
+    module PostProcessor
+      class MailNotification  < Patriot::Command::PostProcessor::Base
+
+        TO_PROP_KEY = 'to'
+        ON_PROP_KEY = 'on'
+
+        declare_post_processor_name :mail_notification 
+
+        def validate_props(props)
+          raise "#{TO_PROP_KEY} is not specified" unless props.has_key?(TO_PROP_KEY)
+          raise "#{ON_PROP_KEY} is not specified" unless props.has_key?(ON_PROP_KEY)
+        end
+
+        def process(cmd, worker, job_ticket)
+          on = @props[ON_PROP_KEY]
+          on = [on] unless on.is_a?(Array)
+          on = on.map{|o| Patriot::Command::ExitCode.value_of(o)}
+          exit_code = job_ticket.exit_code
+          return unless on.include?(exit_code)
+          case exit_code
+          when Patriot::Command::ExitCode::SUCCEEDED then process_success(cmd, worker, job_ticket)
+          when Patriot::Command::ExitCode::FAILED    then process_failure(cmd, worker, job_ticket)
+          end
+        end
+
+        def process_success(cmd, worker, job_ticket)
+          from = worker.config.get(Patriot::Util::Config::ADMIN_USER_KEY)
+          to   = @props[TO_PROP_KEY]
+          subject = "#{job_ticket.job_id} has been successfully finished"
+          body = "#{job_ticket.job_id} has been successfully finished \n\n --- \n #{job_ticket.description}"
+          deliver(from, to, subject, body)
+        end
+
+        def process_failure(cmd, worker, job_ticket)
+          from = worker.config.get(Patriot::Util::Config::ADMIN_USER_KEY)
+          to   = @props[TO_PROP_KEY]
+          subject = "#{job_ticket.job_id} has been failed"
+          body = "#{job_ticket.job_id} has been failed \n\n --- \n #{job_ticket.description}"
+          deliver(from, to, subject, body)
+        end
+
+        def deliver(from_addr, to_addr, msg_subj, msg_body)
+          Mail.deliver do
+            from from_addr
+            to   to_addr
+            subject msg_subj
+            body    msg_body
+          end
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/patriot/command/post_processor/retrial.rb

@@ -0,0 +1,37 @@
+module Patriot
+  module Command
+    module PostProcessor
+      class Retrial < Patriot::Command::PostProcessor::Base
+
+        COUNT_PROP_KEY    = 'count'
+        INTERVAL_PROP_KEY = 'interval'
+
+        declare_post_processor_name :retrial
+
+        def validate_props(props)
+          raise "#{COUNT_PROP_KEY} is not specified" unless props.has_key?(COUNT_PROP_KEY)
+          raise "#{INTERVAL_PROP_KEY} is not specified" unless props.has_key?(INTERVAL_PROP_KEY)
+        end
+
+        def process_failure(cmd, worker, job_ticket)
+          found = false
+          cmd.post_processors.each do |pp|
+            next unless pp.is_a?(Patriot::Command::PostProcessor::Retrial)
+            raise "multiple retry processors in #{cmd.job_id}" if found
+            found = true
+            # count first attempt in
+            pp.props[COUNT_PROP_KEY] = pp.props[COUNT_PROP_KEY] - 1
+            return if pp.props[COUNT_PROP_KEY] == 0 
+            cmd.start_datetime = Time.now + pp.props[INTERVAL_PROP_KEY]
+          end
+          job = cmd.to_job
+          job[Patriot::Command::STATE_ATTR] = Patriot::JobStore::JobState::WAIT
+          worker.job_store.register(Time.now.to_i, [job])
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/patriot/command/post_processor/skip_on_fail.rb

@@ -0,0 +1,18 @@
+module Patriot
+  module Command
+    module PostProcessor
+
+      class SkipOnFail < Patriot::Command::PostProcessor::Base
+
+        declare_post_processor_name :skip_on_fail
+
+        def process_failure(cmd, worker, job_ticket)
+          worker.job_store.set_state(Time.now.to_i, [cmd.job_id], Patriot::JobStore::JobState::SUCCEEDED)
+        end
+
+      end
+    end
+  end
+end
+
+

data/lib/patriot/command/sh_command.rb

@@ -10,7 +10,7 @@ module Patriot
       command_attr :commands do |cmd, a, v|
         cmd.commands = v.is_a?(Array)? v : [v]
       end
-      volatile_attr :name, :name_suffix
+      command_attr :name, :name_suffix
       validate_existence :name 
 
       # @see Patriot::Command::Base#job_id

data/lib/patriot/job_store.rb

@@ -40,9 +40,8 @@ module Patriot
     # mapping from exit code to job state
     # @see Patriot::Command::ExitCode
     EXIT_CODE_TO_STATE = {
-      Patriot::Command::ExitCode::SUCCEEDED       => Patriot::JobStore::JobState::SUCCEEDED,
-      Patriot::Command::ExitCode::FAILURE_SKIPPED => Patriot::JobStore::JobState::SUCCEEDED,
-      Patriot::Command::ExitCode::FAILED          => Patriot::JobStore::JobState::FAILED
+      Patriot::Command::ExitCode::SUCCEEDED => Patriot::JobStore::JobState::SUCCEEDED,
+      Patriot::Command::ExitCode::FAILED    => Patriot::JobStore::JobState::FAILED
     }
 
     # a prefix for configuration key

data/lib/patriot/job_store/job.rb

@@ -43,7 +43,7 @@ module Patriot
       def read_command(command)
         Patriot::Command::COMMON_ATTRIBUTES.each do |attr|
           value = command.instance_variable_get("@#{attr}".to_sym)
-          self[attr] = value unless value.nil?
+          self[attr] = _to_stdobj(value) unless value.nil?
         end
         _to_stdobj(command).each{|k,v| self[k] = v}
       end
@@ -60,6 +60,13 @@ module Patriot
             hash[attr.to_s] = _to_stdobj(value) unless value.nil?
           end
           return hash
+        elsif obj.is_a?(Patriot::Command::PostProcessor::Base)
+          hash = {}
+          hash[Patriot::Command::PostProcessor::POST_PROCESSOR_CLASS_KEY] = obj.class.to_s.gsub(/::/, '.')
+          obj.props.each do |k,v|
+            hash[k.to_s] = _to_stdobj(v) unless v.nil?
+          end
+          return hash
         elsif obj.is_a?(Hash)
           hash = {}
           obj.each{|k,v| hash[k.to_s] = _to_stdobj(v)}
@@ -94,6 +101,10 @@ module Patriot
               cmd.instance_variable_set("@#{k}".to_sym, _from_stdobj(v, config))
             end
             return cmd
+          elsif obj.has_key?(Patriot::Command::PostProcessor::POST_PROCESSOR_CLASS_KEY)
+            cmd_cls = obj.delete(Patriot::Command::PostProcessor::POST_PROCESSOR_CLASS_KEY)
+            cmd_cls = cmd_cls.split('.').inject(Object){|c,name| c.const_get(name)}
+            return cmd_cls.new(obj)
           else
             hash = {}
             obj.each{|k,v| hash[k] = _from_stdobj(v, config)}

data/lib/patriot/tool/batch_parser.rb

@@ -48,7 +48,9 @@ module Patriot
       # @return an array of commands
       def parse(date, files, options = {}, &blk)
         return if files.empty?
-        datetime = DateTime.parse(date)
+        ds = date.split('-')
+        raise "illegal format of date #{date}" unless ds.size == 3
+        datetime = Time.new(ds[0], ds[1], ds[2])
         # for backward compatibility to be removed
         $dt    = date
         $month = date.split('-').values_at(0,1).join('-')

data/lib/patriot/util/config.rb

@@ -17,6 +17,8 @@ module Patriot
       PLUGIN_LIB_DIR = 'lib' 
       # plugin initiation script
       PLUGIN_INIT_SCRIPT = 'init.rb'
+      # admin user mail address
+      ADMIN_USER_KEY = 'admin_user'
 
       # load configuration file
       # @param option [Hash]

data/lib/patriot/util/cron_format_parser.rb

@@ -21,14 +21,16 @@ module Patriot
       # the list of minutes
       MINUTES = 0.upto(59).to_a
 
+      DAY_IN_SECOND = 60 * 60 * 24
+
       # expand a given date to the array of time which should be executed in case of a given cron field
-      # @param date [DateTime] target datetime
+      # @param date [Time] target datetime
       # @param cron_field [String] interval in cron format
-      # @return [Array<DateTime>] a list of datetime which match the cron format
+      # @return [Array<Time>] a list of datetime which match the cron format
       def expand_on_date(date, cron_field = nil)
         cron_field = DEFAULT_CRON_FIELD if cron_field.nil? || cron_field.empty?
         if cron_field == END_OF_EVERY_MONTH
-          return date.next.day == 1 ? [date] : []
+          return (date + DAY_IN_SECOND).day == 1 ? [date] : []
         end
         field_splits = cron_field.split
         raise "illegal cron field format #{cron_field}" unless field_splits.size == 5
@@ -36,13 +38,13 @@ module Patriot
         return [] unless is_target_day?(date, day, month, week)
         return target_hours(hour).map do |h|
           target_minutes(minute).map do |m|
-            DateTime.new(date.year, date.month, date.day, h, m, 0)
+            Time.new(date.year, date.month, date.day, h, m, 0)
           end
         end.flatten
       end
 
       # check a given date is a target or not
-      # @param date [DateTime] a datetime to be checked
+      # @param date [Time] a datetime to be checked
       # @param day [String] day field in cron format
       # @param month [String] month field in cron format
       # @param week [String] week field in cron format

data/lib/patriot/util/date_util.rb

@@ -35,7 +35,7 @@ module Patriot
         min = min%60
         hour = hour%24
    
-        new_dt = DateTime.new(year, month, day, hour, min,sec)
+        new_dt = DateTime.new(year, month, day, hour, min, sec)
         new_dt = new_dt >> diff[:month]
         new_dt = new_dt + diff[:day]
         return new_dt.strftime(fmt)

data/lib/patriot/worker/base.rb

@@ -12,7 +12,7 @@ module Patriot
       include Patriot::Util::Retry
       include Patriot::JobStore::Factory
         
-      attr_accessor :host, :status, :cycle, :job_store
+      attr_accessor :host, :status, :cycle, :job_store, :config
 
       # @param config [Patriot::Util::Config::Base]
       def initialize(config)
@@ -47,7 +47,7 @@ module Patriot
         @logger.info " executing job: #{job_ticket.job_id}"
         command                 = response[:command]
         job_ticket.execution_id = response[:execution_id]
-        job_ticket.exit_code    = command.skip_on_fail? ? Patriot::Command::ExitCode::FAILURE_SKIPPED : Patriot::Command::ExitCode::FAILED
+        job_ticket.exit_code    = Patriot::Command::ExitCode::FAILED
         begin
           command.execute
           job_ticket.exit_code  = Patriot::Command::ExitCode::SUCCEEDED
@@ -62,6 +62,17 @@ module Patriot
           rescue Exception => job_store_error
             @logger.error job_store_error
           end
+          unless command.post_processors.nil?
+            command.post_processors.each do |pp|
+              begin
+                @logger.info "executing post process by #{pp}"
+                pp.process(command, self, job_ticket)
+              rescue Exception => post_process_error
+                @logger.error "post process by #{pp} failed"
+                @logger.error post_process_error
+              end
+            end
+          end
         end
         return job_ticket.exit_code
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: patriot-workflow-scheduler
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Teruyoshi Zenmyo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-31 00:00:00.000000000 Z
+date: 2015-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -122,6 +122,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: mail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
 - !ruby/object:Gem::Dependency
   name: tilt
   requirement: !ruby/object:Gem::Requirement
@@ -153,6 +167,11 @@ files:
 - lib/patriot/command/command_macro.rb
 - lib/patriot/command/composite.rb
 - lib/patriot/command/parser.rb
+- lib/patriot/command/post_processor.rb
+- lib/patriot/command/post_processor/base.rb
+- lib/patriot/command/post_processor/mail_notification.rb
+- lib/patriot/command/post_processor/retrial.rb
+- lib/patriot/command/post_processor/skip_on_fail.rb
 - lib/patriot/command/sh_command.rb
 - lib/patriot/controller.rb
 - lib/patriot/controller/package_controller.rb