patriot-workflow-scheduler 0.6.1

data/lib/patriot/util/logger/webrick_log_factory.rb

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'webrick'
+require 'singleton'
+require 'patriot/util/logger/factory'
+
+module Patriot
+  module Util
+    module Logger
+      # a logger factory implementation based on Webrick Logger
+      class WebrickLogFactory < Patriot::Util::Logger::Factory
+        # configuration key for log file
+        LOG_FILE_KEY  = :log_file
+        # configuration key for log level
+        LOG_LEVEL_KEY = :log_level
+
+        include Singleton
+
+        # @see Patriot::Util::Logger::Factory
+        def build(name, config)
+          log_file  = get_log_file(config)
+          log_level = get_log_level(config)
+          logger = WEBrick::BasicLog.new(log_file, log_level)
+          return logger
+        end
+        private :build
+
+        # @param config [Patriot::Util::Config::Base]
+        # @return [String] path to the log file
+        def get_log_file(config)
+          log_file = config.get(LOG_FILE_KEY)
+          return log_file
+        end
+        private :get_log_file
+
+        # get log level from configuration
+        # @param config [Patriot::Util::Config::Base]
+        def get_log_level(config)
+          log_level = config.get(LOG_LEVEL_KEY)
+          const     = "WEBrick::BasicLog::#{log_level}".to_sym
+          return const 
+        end
+        private :get_log_level
+
+      end
+    end
+  end
+end

data/lib/patriot/util/param.rb

@@ -0,0 +1,73 @@
+module Patriot
+  module Util
+    # namespace for parameter handling functions
+    module Param
+
+      # replace parameter values in command attribute valeus
+      # @param attr_val an attribute value to be evaluated
+      def eval_attr(attr_val)
+        if attr_val.is_a?(Hash)
+          entries = {}
+          attr_val.each{|k,v| entries[eval_attr(k)] = eval_attr(v) }
+          return entries
+        elsif attr_val.is_a?(Array)
+          return attr_val.map{|e| eval_attr(e)}
+        elsif attr_val.is_a?(String)
+          return eval_string_attr(attr_val)
+        else
+          # only evaluate attributes in String
+          return attr_val
+        end
+      end
+
+      # evaluate variables in a string expression
+      # @param str [String] a string expression to be evaluated
+      # @param vars [Hash] variables used in the evaluation
+      # @return [String] a evaluated string expression
+      def eval_string_attr(str, vars = {})
+        s = StringScanner.new(str)
+        s.scan(/(.*?)\#\{/m)
+        # retrun immediatelly if variables are not contained
+        return str unless s.matched?
+        prefix = s[1]
+        nest = 1 # depth of parenthesis
+        var = "" # variable expression
+        prev_rest = s.rest
+        while nest > 0
+          tmp = s.scan(/(.*?)[\{\}]/m) # for hash objects, etc
+          if s.matched?
+            if /.*?\{/ =~ tmp
+              nest = nest + 1
+              var << s[0]
+            else 
+              nest = nest - 1
+              if nest >  0
+                var << s[0]
+              else
+                # does not include the last parenthesis indicates end of the variable 
+                var << s[1]
+              end
+            end
+          end
+          raise "infinte loop #{str} : rest #{s.rest} : #{nest}" if prev_rest == s.rest
+          prev_rest = s.rest
+        end
+        # evaluate the variable
+        var_binding = build_var_binding(vars)
+        var_binding = binding if var_binding.nil?
+        evaled_var = eval var, var_binding
+        # farther variables are handled by next invocation
+        return "#{prefix}#{evaled_var}#{eval_string_attr(prev_rest, vars)}"
+      end
+
+      def build_var_binding(vars)
+        return nil if vars.empty?
+        raise "illegal key var exist in #{vars.inspect}" if vars.has_key?('vars')
+        assign_exps = [vars.map{|k,v| "#{k} = vars['#{k}']"}] | ["binding"]
+        return eval assign_exps.join(";")
+      end
+      private :build_var_binding
+
+    end
+  end
+end

data/lib/patriot/util/retry.rb

@@ -0,0 +1,30 @@
+
+module Patriot
+  module Util
+    # a mudule enables operations to be retried
+    module Retry
+      # execute block and retry the block 
+      # @param retry_config :num_retry the max number of retry, :wait_time retry interval
+      # @raise an exception thrown from the block
+      # @return return value of the block
+      def execute_with_retry(retry_config = {}, &blk)
+        retry_config = {:num_retry => 3, :wait_time => 3}.merge(retry_config)
+        e = nil
+        1.upto(retry_config[:num_retry]) do |i|
+          begin
+            return yield
+          rescue Exception => e
+            if @logger
+              @logger.error "fail to execute (#{i}) #{blk.to_s}"
+              @logger.error e
+              $@.each{|m| @logger.error m}
+            end
+          end 
+          sleep retry_config[:wait_time]
+        end
+        raise e unless e.nil?
+      end
+      module_function :execute_with_retry
+    end
+  end
+end

data/lib/patriot/util/script.rb

@@ -0,0 +1,52 @@
+module Patriot
+  module Util
+    # a module to find target files
+    module Script
+      include Patriot::Util::DateUtil
+
+      # get target batch files from a given path
+      # @param path [String] path to target directory
+      # @param date [String] target date in '%Y-%m-%d'
+      # @param opt [Hash]
+      # @option opt :all [Boolean] force target all files
+      # @return [Array<String>] a list of target files
+      def get_batch_files(path, date, opt = {})
+        return [path] if File.file?(path) && File.extname(path) == ".pbc" 
+        files = []
+        opt   = target_option(date, opt)
+        files = Dir.glob("#{path}/**/*.pbc").find_all do |file|
+          target_file?(file, opt)
+        end
+        return files
+      end
+
+      def target_option(date, opt = {})
+        opt = {:all => false}.merge(opt)
+        unless opt[:all]
+          d = date.split('-')
+          opt[:day] = true unless opt.has_key?(:day)
+          unless opt.has_key?(:month)
+            opt[:month] = date_add(date,1) =~ /[\d]{4}-[\d]{2}-01/ ? true : false 
+          end
+          unless opt.has_key?(:week)
+            opt[:week]  = Date.new(d[0].to_i, d[1].to_i, d[2].to_i).wday
+          end
+        end
+        return opt
+      end
+      private :target_option
+
+      def target_file?(file, options)
+        case
+        when options[:all]                            then true
+        when file =~ /\/daily\// && options[:day]     then true
+        when file =~ /\/monthly\// && options[:month] then true
+        when file =~ /\/weekly\/#{options[:week]}\//  then true
+        else false
+        end
+      end
+      private :target_file?
+
+    end
+  end
+end

data/lib/patriot/util/system.rb

@@ -0,0 +1,120 @@
+require 'fileutils'
+require 'time'
+module Patriot
+  module Util
+    # module for interaction with OS
+    module System
+      # exception in case of OS command failed
+      class ExternalCommandException < Exception; end
+
+      # suffix of file where stdout is written
+      STDOUT_SUFFIX=".stdout"
+      # suffix of file where stderr is written
+      STDERR_SUFFIX=".stderr"
+
+      # configuration key for tmp directory where stdout/stderr are written
+      PATRIOT_TMP_DIR_KEY        = "patriot.tmp.dir"
+      # defatul path to the tmp directory
+      DEFAULT_PATRIOT_TMP_DIR    = "/tmp/patriot-workflow-scheduler"
+      # max size of error message included in exceptions
+      MAX_ERROR_MSG_SIZE_KEY     = "patriot.max.error.size"
+      # defaut max size of error message
+      DEFAULT_MAX_ERROR_MSG_SIZE = 256
+
+      # @private
+      # get path to tmp directory (visible for test)
+      def tmp_dir(pid, dt, ts, tmp_dir = DEFAULT_PATRIOT_TMP_DIR)
+        prefix = "p#{pid.to_s}"
+        prefix = "j#{Thread.current[Patriot::Worker::JOB_ID_IN_EXECUTION]}" if Thread.current[Patriot::Worker::JOB_ID_IN_EXECUTION] 
+        ts_exp = Time.at(ts).strftime("%Y%m%d_%H%M%S")
+        return File.join(tmp_dir, dt, "#{prefix}_#{ts_exp}")
+      end
+
+      # @private
+      # fork and execute the command (visible for test)
+      def do_fork(cmd, dt, ts, tmp_dir = DEFAULT_PATRIOT_TMP_DIR)
+        cid = fork do 
+          tmpdir = tmp_dir($$, dt, ts, tmp_dir)
+          FileUtils.mkdir_p(tmpdir, {:mode => 0777})
+          std_out = File.join(tmpdir, "#{$$.to_i}#{STDOUT_SUFFIX}")
+          std_err = File.join(tmpdir, "#{$$.to_i}#{STDERR_SUFFIX}")
+          STDOUT.reopen(std_out,"w")
+          STDERR.reopen(std_err,"w")
+          exec(cmd) 
+        end
+        return cid
+      end
+
+      # execute command on OS
+      # @param command [String] command to be executed
+      # @yield block for error handling
+      def execute_command(command, &blk)
+        so, se = nil
+
+        time_obj = Time.now
+        ts       = time_obj.to_i
+        dt       = time_obj.strftime("%Y-%m-%d")
+
+        tmp_dir_base  = @config.get(PATRIOT_TMP_DIR_KEY, DEFAULT_PATRIOT_TMP_DIR)
+
+        # the forked variable is used for checking whether fork invocation hangs.
+        #  (due to https://redmine.ruby-lang.org/issues/5240 ?)
+        forked = false
+        until forked
+          cid = do_fork(command, dt, ts, tmp_dir_base)
+          tmpdir = tmp_dir(cid, dt, ts, tmp_dir_base)
+          i = 0
+          # If fork hangs, output directory would not be created.
+          # wait at most 5 seconds for the directory created.
+          until forked || i > 5
+            sleep(1)
+            forked = File.exist?(tmpdir)
+            i = i+1
+          end
+          # fork hanged, kill the hanged process.
+          unless forked
+            # check whether cid is id of child process to avoid to kill unrelated processes
+            begin
+              if Process.waitpid(cid, Process::WNOHANG).nil?
+                @logger.warn("forked process :#{cid} hanged. kill #{cid}")
+                Process.kill("KILL", cid)
+                @logger.warn("SIGKILL sent to #{cid}")
+                Process.waitpid(cid)
+                @logger.warn("#{cid} is killed")
+              else
+                raise ExternalCommandException, "#{cid} is not a child of this"
+              end
+            rescue Exception => e
+              @logger.warn "failed to kill hanged process #{cid}"
+              raise e
+            end
+          end
+        end
+
+        @logger.info "executing #{command}: results stored in #{tmpdir}" 
+        pid, status = Process.waitpid2(cid)
+        so = File.join(tmpdir, "#{cid.to_i}#{STDOUT_SUFFIX}")
+        se = File.join(tmpdir, "#{cid.to_i}#{STDERR_SUFFIX}")
+
+        @logger.info "#{command} is finished" 
+        return so if status.exitstatus == 0
+        @logger.warn "#{command} end with exit status #{status.exitstatus}" 
+        if block_given?
+          yield(status, so, se)
+        else
+          max_err_size = @config.get(MAX_ERROR_MSG_SIZE_KEY, DEFAULT_MAX_ERROR_MSG_SIZE)
+          err_size = File.stat(se).size
+          err_msg =  "#{command}\n#{se} :"
+          if err_size < max_err_size
+            File.open(se){|f| err_msg = "#{err_msg}\n#{f.read}"}
+          else
+            err_msg = "#{err_msg} \n the size of stderr is #{err_size} (> #{max_err_size}" 
+          end
+          raise ExternalCommandException, err_msg
+        end
+      end
+
+    end
+  end
+end
+

data/lib/patriot/worker.rb

@@ -0,0 +1,35 @@
+module Patriot
+  # namespace for workers
+  module Worker
+    # states of worker
+    module Status
+      # worker is active
+      ACTIVE   = 'ACTIVE'
+      # worker sleeps (don't execute jobs)
+      SLEEP    = 'SLEEP'
+      # worker is going to down
+      SHUTDOWN = 'SHUTDOWN'
+    end
+
+    # name of thread local variable for job_id executed by the thread
+    JOB_ID_IN_EXECUTION          = :JOB_ID_IN_EXECUTION
+
+    # SIGNAL used for graceful shutdown
+    SIGNAL_FOR_GRACEFUL_SHUTDOWN = ['INT', 'TERM']
+    # SIGNAL used for getting thread dump
+    SIGNAL_FOR_THREAD_DUMP       = ['QUIT']
+
+    # default fetch cycle for getting job tickets in minutes
+    DEFAULT_FETCH_CYCLE = 300
+    # default limit of the number of jobs fetched at once
+    DEFAULT_FETCH_LIMIT = 200
+    # default worker name (for logging)
+    DEFAULT_WORKER_NAME = 'default'
+
+    require 'patriot/worker/servlet'
+    require 'patriot/worker/info_server'
+    require 'patriot/worker/base'
+    autoload :MultiNodeWorker, 'patriot/worker/multi_node_worker'
+    autoload :JobStoreServer, 'patriot/worker/job_store_server'
+  end
+end

data/lib/patriot/worker/base.rb

@@ -0,0 +1,153 @@
+require 'date'
+require 'patriot/command'
+
+module Patriot
+  module Worker
+
+    # @abstract
+    # base class for worker implementations
+    class Base
+      
+      include Patriot::Util::Logger
+      include Patriot::Util::Retry
+      include Patriot::JobStore::Factory
+        
+      attr_accessor :host, :status, :cycle, :job_store
+
+      # @param config [Patriot::Util::Config::Base]
+      def initialize(config)
+        raise "configuration is nil" if config.nil?
+        @logger      = create_logger(config)
+        @config      = config
+        @job_store   = create_jobstore(Patriot::JobStore::ROOT_STORE_ID, @config)
+        @host        = `hostname`.chomp
+        @cycle       = config.get('fetch_cycle', Patriot::Worker::DEFAULT_FETCH_CYCLE).to_i
+        @fetch_limit = config.get('fetch_limit', Patriot::Worker::DEFAULT_FETCH_LIMIT).to_i
+        @worker_name = config.get('worker_name', Patriot::Worker::DEFAULT_WORKER_NAME)
+        @pid_file    = File.join($home, 'run', "patriot-worker_#{@worker_name}.pid")
+        @info_server = Patriot::Worker::InfoServer.new(self,@config)
+      end
+
+      # execute a job
+      # @param [Patriot::JobStore::JobTicket] job_ticket a ticket of job to be executed
+      # @return [Patriot::Command::ExitCode]
+      def execute_job(job_ticket)
+        job_ticket.exec_host   = @host
+        job_ticket.exec_node   = Thread.current[:name]
+        begin
+          response = @job_store.offer_to_execute(job_ticket)
+        rescue Exception => e
+          @logger.error e
+          return Patriot::Command::ExitCode::FAILED
+        end
+
+        # already executed by other node
+        return Patriot::Command::ExitCode::SKIPPED if response.nil?
+
+        @logger.info " executing job: #{job_ticket.job_id}"
+        command                 = response[:command]
+        job_ticket.execution_id = response[:execution_id]
+        job_ticket.exit_code    = command.skip_on_fail? ? Patriot::Command::ExitCode::FAILURE_SKIPPED : Patriot::Command::ExitCode::FAILED
+        begin
+          command.execute
+          job_ticket.exit_code  = Patriot::Command::ExitCode::SUCCEEDED
+        rescue Exception => e
+          @logger.warn " job : #{job_ticket.job_id} failed"
+          job_ticket.description = e.to_s
+        else
+          job_ticket.description = command.description
+        ensure
+          begin
+            execute_with_retry{ @job_store.report_completion_status(job_ticket) }
+          rescue Exception => job_store_error
+            @logger.error job_store_error
+          end
+        end
+        return job_ticket.exit_code
+      end
+
+      # @return [Integer] pid if the worker is running, otherwise nil
+      def get_pid
+        return nil unless File.exists?(@pid_file)
+        pid = nil
+        File.open(@pid_file,'r'){|f| pid = f.read.strip.to_i }
+        begin
+          Process.getpgid(pid)
+        rescue Errno::ESRCH
+          @logger.warn("process #{pid} not exist but pid file remains")
+          return nil
+        end
+        return pid
+      end
+
+      # send a request graceful shutdown to a running worker
+      # @return [Boolean] true worker is running and request is sent, otherwise false
+      def request_shutdown
+        pid = get_pid
+        if pid.nil?
+          @logger.info("worker #{@worker_name} does not exist")
+          return false
+        end
+        Process.kill(SIGNAL_FOR_GRACEFUL_SHUTDOWN[0], pid.to_i)
+        return true
+      end
+
+      # main entry point of worker processing
+      def start_worker
+        return unless get_pid.nil?
+
+        @logger.info "starting worker #{@node}@#{@host}"
+        File.open(@pid_file, 'w') {|f| f.write($$)} # save pid for shutdown
+        set_traps
+        @info_server.start_server
+        @logger.info "initiating worker #{@node}@#{@host}"
+        init_worker
+        @status = Patriot::Worker::Status::ACTIVE
+        @logger.info "start worker #{@node}@#{@host}"
+        run_worker
+        @logger.info "shutting down worker #{@node}@#{@host}"
+        stop_worker
+        # should be last since worker_admin judge availability from the info_server
+        @info_server.shutdown_server
+      end
+
+
+      # should be overrided in sub class
+      # This method is for implementation-specific configuration
+      def init_worker 
+        raise NotImplementedError
+      end
+
+      # should be overrided in sub class
+      # Main loop in which the worker fetches and executes jobs should be implemented here
+      def run_worker
+        raise NotImplementedError
+      end
+
+      # should be overrided in sub class
+      # Tasks for tearing down the worker should be implemented here
+      def stop_worker
+        raise NotImplementedError
+      end
+
+      def set_traps
+        Patriot::Worker::SIGNAL_FOR_GRACEFUL_SHUTDOWN.each do |s|
+          Signal.trap(s) do
+            @logger.info "SIG#{s}: worker will terminate"
+            @status = Patriot::Worker::Status::SHUTDOWN
+          end
+        end
+        Patriot::Worker::SIGNAL_FOR_THREAD_DUMP.each do |s|
+          Signal.trap(s) do
+            # TODO output to separated stream
+            Thread.list.each do |t|
+              @logger.info("Thread #{t[:name]}\n#{t.backtrace.map{|bt| "\t#{bt}"}.join("\n")}")
+            end
+          end
+        end
+      end
+      private :set_traps
+
+    end
+  end
+end