patriot-workflow-scheduler 0.6.1

data/lib/patriot/command/composite.rb

@@ -0,0 +1,49 @@
+module Patriot
+  module Command
+    # a command which is composed of multiple sub commands
+    class CompositeCommand < Patriot::Command::CommandGroup 
+      declare_command_name :composite_command 
+      declare_command_name :composite_job 
+      private_command_attr :contained_commands => []
+      volatile_attr :name, :name_suffix
+
+      # @return [String] the identifier of this composite command
+      # @see Patriot::Command::Base#job_id
+      def job_id  
+        return "#{command_name}_#{@name}_#{@name_suffix}"
+      end
+
+      # @see Patriot::Command::Base#description
+      def description
+        first_job = @contained_commands.first
+        first_job = first_job.description unless first_job.nil?
+        return "#{first_job} ... (#{@contained_commands.size} jobs)"
+      end
+
+      # configure this composite command.
+      # pull up required/produced products from the sub commands
+      # @see Patriot::Command::Base#configure
+      def configure
+        @name_suffix ||= _date_
+        # don't do flatten to handle nested composite commands
+        @subcommands.map do |cmd|
+           cmd.build(@param).each do |cmd|
+            require cmd['requisites']
+            produce cmd['products']
+            @contained_commands << cmd
+          end
+        end
+        return self
+      end
+
+      # execute the contained commands
+      # @see Patriot::Command::Base#execute
+      def execute
+        @contained_commands.each do |c|
+          c.execute
+        end
+      end 
+
+    end
+  end
+end

data/lib/patriot/command/parser.rb

@@ -0,0 +1,78 @@
+require "erb"
+module Patriot
+  module Command
+    
+    # namespace for DSL parser
+    module Parser
+
+      # create a command
+      # @param cls [Class] class of the command
+      # @yield block to set attributes to the command
+      def new_command(cls, &blk)
+        raise "configuration is not set" if @config.nil?
+        command = cls.new(@config)
+        command.target_datetime = @target_datetime
+        @macros.each{|n,b| command.batch_macro(n,&b)} unless @macros.nil?
+        command.instance_eval(&blk) if block_given?
+        return command
+      end
+
+      # parse DSL by processing the DSL description as block
+      # @param datetime [DateTime] the datetime for which the job works
+      # @return a list of command defined in the DSL description
+      def parse(datetime, blk)
+        self.target_datetime = datetime
+        self.instance_eval(blk)
+        return self.build({})
+      end
+
+      # load macro to be able to be used in the DSL
+      # @param macro_file [String] path to a macro file
+      def load_macro(macro_file)
+        macro_file = File.expand_path(macro_file,$home)
+        @logger.info "loading macro file #{macro_file}" 
+        open(macro_file){|f|
+          self.instance_eval(f.read)
+        }
+      end
+
+      # define macro.
+      # this method is used in macro files
+      # @param name name of the macro
+      # @yieldreturn the result of the macro evaluation
+      def batch_macro(name, &blk)
+        raise "#{name} is invalid macro name (duplicate or defined method)" if respond_to?(name.to_sym)
+        eigenclass = class << self
+          self
+        end
+        @macros[name] = blk
+        eigenclass.send(:define_method, name, &blk)
+      end
+        
+      # import an ERB template
+      # @param file [String] path to the ERB template
+      # @param _vars [Hash] a hash from variable name to its value used in the ERB
+      def import_erb_config(file, _vars)
+        file = File.expand_path(file,$home)
+        # set variables
+        erb = _vars.map{|k,v| "<%#{k} = #{v.inspect}%>"}.join("\n")
+        erb << "\n"
+
+        # read the ERB
+        exp=""
+        open(file) do |f|
+          erb << f.read
+          exp = ERB.new(erb).result(binding)
+        end
+        begin
+          self.instance_eval(exp)
+        rescue => e
+          @logger.error("failed to parse #{file}")
+          @logger.error(erb)
+          raise e
+        end
+      end
+    end
+  end
+end
+

data/lib/patriot/command/sh_command.rb

@@ -0,0 +1,42 @@
+module Patriot
+  module Command
+    # a command which executes shell scripts
+    class ShCommand < Patriot::Command::Base
+      include Patriot::Util::System
+
+      declare_command_name :sh
+
+      command_attr :connector => '&&'
+      command_attr :commands do |cmd, a, v|
+        cmd.commands = v.is_a?(Array)? v : [v]
+      end
+      volatile_attr :name, :name_suffix
+      validate_existence :name 
+
+      # @see Patriot::Command::Base#job_id
+      def job_id  
+        return "#{command_name}_#{@name}_#{@name_suffix}"
+      end
+
+      # @see Patriot::Command::Base#configure
+      def configure
+        @name_suffix ||= _date_
+        return self
+      end
+
+      # @see Patriot::Command::Base#description
+      def description
+        return @commands.join(@connector)
+      end
+
+      # @see Patriot::Command::Base#execute
+      def execute
+        @logger.info "start shell command "
+        @commands.each do |c|
+          execute_command(c)
+        end
+        @logger.info "end shell command "
+      end
+    end
+  end
+end

data/lib/patriot/controller.rb

@@ -0,0 +1,2 @@
+require 'patriot/controller/package_controller'
+require 'patriot/controller/worker_admin_controller'

data/lib/patriot/controller/package_controller.rb

@@ -0,0 +1,81 @@
+require 'rubygems/installer'
+require 'rubygems/version_option'
+require 'rubygems/remote_fetcher'
+require 'rubygems/dependency_installer'
+require 'fileutils'
+
+module Patriot
+  # a name space for controllers which encapsulate complicated operations
+  module Controller
+    # Controller class for remote management of workers
+    class PackageController
+      include Patriot::Util::Config
+      include Patriot::Util::Logger
+
+      # constructor
+      # @param config [Patriot::Util::Config::Base] configuration of this controller
+      def initialize(config)
+        @config = config
+        @logger = create_logger(config)
+        @plugin_dir = config.get(Patriot::Util::Config::PLUGIN_DIR_KEY, Patriot::Util::Config::DEFAULT_PLUGIN_DIR)
+        @plugin_dir = File.expand_path(@plugin_dir, $home)
+      end
+
+      # upgrade deployment
+      def upgrade(pkg = 'patriot-workflow-scheduler')
+        # upgrade plugins
+        plugins = @config.get(Patriot::Util::Config::PLUGIN_KEY, [])
+        plugins = [plugins] unless plugins.is_a?(Array)
+        plugins.each{|plugin| install_plugin(plugin, {:force => true})}
+
+        # upgrade core package
+        dependency = Gem::Dependency.new(pkg || 'patriot-workflow-scheduler')
+        path = dependency.name =~ /\.gem$/i ? dependency.name : Gem::RemoteFetcher.fetcher.download_to_cache(dependency) 
+        installed_dir = Gem::Installer.new(path).dir
+        installer = Gem::DependencyInstaller.new
+        @logger.info "upgrade to #{dependency}"
+        installer.install path
+
+        public_dir = File.join(installed_dir, "skel", "public")
+        @logger.info "copy #{public_dir} to  #{$home}"
+        FileUtils.cp_r(public_dir, $home)
+        FileUtils.cp(File.join(installed_dir, 'bin', 'patriot'), File.join($home, 'bin', 'patriot'))
+
+      end
+
+      # install plugin to plugin directory
+      # @param [String] plugin name of the plugin
+      # @param opts
+      # @option [Boolean] :force set true to overwrite the installation
+      def install_plugin(plugin, opts = {})
+        @logger.info "install #{plugin}"
+        dependency = Gem::Dependency.new plugin, opts[:version]
+        path = dependency.name =~ /\.gem$/i ? dependency.name : Gem::RemoteFetcher.fetcher.download_to_cache(dependency) 
+        raise "Gem '#{plugin}' not fetchable."  unless path
+        basename = File.basename path, '.gem'
+        # remvoe version
+        basename = basename.gsub(/-[\d]+\.[\d]+\.[\d]+$/, "")
+        target_dir = File.join(@plugin_dir, basename)
+        if opts[:force] == true
+          @logger.info "remove old #{target_dir}"
+          FileUtils.rm_r target_dir if File.exist?(target_dir)
+        else
+          raise "#{target_dir} alrady exist" if File.exist?(target_dir)
+        end
+        FileUtils.mkdir_p target_dir
+        if opts[:unpack]
+          installer = Gem::Installer.new(path, :unpack=>true)
+          installer.unpack target_dir
+        else
+          installer = Gem::DependencyInstaller.new
+          installer.install path
+          installed_dir = Gem::Installer.new(path).dir
+          init_rb = File.join(installed_dir, "init.rb")
+          FileUtils.cp(init_rb, target_dir)
+        end
+        @logger.info "#{path} installed: #{target_dir}'"
+      end
+
+    end
+  end
+end

data/lib/patriot/controller/worker_admin_controller.rb

@@ -0,0 +1,159 @@
+require 'rest_client'
+
+module Patriot
+  module Controller
+    # Controller class for remote management of workers
+    class WorkerAdminController
+      include Patriot::Util::Config
+      include Patriot::Util::Logger
+
+      # a command line used for start/stop workers
+      WORKER_COMMAND = File.join($home || Dir.pwd,'bin', 'patriot worker')
+      # a command line used for upgrade
+      UPGRADE_COMMAND = File.join($home || Dir.pwd,'bin', 'patriot upgrade')
+
+      # constructor
+      # @param config [Patriot::Util::Config::Base] configuration of this controller
+      def initialize(config)
+        @config = config
+        @logger = create_logger(config)
+        set_default_values
+      end
+
+      # @private
+      def set_default_values
+        @default_hosts = @config.get('worker_hosts') || []
+        @default_port  = @config.get('info_server_port')
+        @user          = @config.get('admin_user')
+      end
+      private :set_default_values
+
+      # execute block for each target hosts
+      # @param options [Hash]
+      # @option options :host  a target host
+      # @option options :hosts a comman separated value of target hosts
+      # @option options :all   set true to target all hosts in the configuration 
+      # @return [Hash] a hash from host name to the result of the block
+      def request_to_target_hosts(options = {}, &blk)
+        hosts = []
+        port = options.has_key?(:port) ? options[:port] : @default_port
+        if options.has_key?(:host)
+          hosts = [options[:host]]
+        elsif options.has_key?(:hosts)
+          hosts = options[:hosts]
+          hosts = hosts.split(",") unless hosts.is_a?(Array)
+        elsif options[:all] == true
+          hosts = @default_hosts
+          hosts = [hosts] unless hosts.is_a?(Array)
+        else
+          raise "any host is not set"
+        end
+        results = {}
+        hosts.each{|h| results[h] = yield(h,port) }
+        return results
+      end
+
+      # get status of a worker or workers
+      # @param options @see {#request_to_target_hosts}
+      # @return [Hash]
+      #   status of worker in String.
+      #   nil for an unresponsive worker
+      def status(options = {})
+        return request_to_target_hosts(options){|h,p| get_worker_status(h,p)}
+      end
+
+      # get status of a worker
+      # @param host [String] host name of the target host
+      # @param port [String] port number of the worker process on the target host
+      # @return [String] status of the server @see {Patriot::Worker::Base}
+      def get_worker_status(host, port)
+        begin
+          return RestClient.get("http://#{host}:#{port}/worker")
+        rescue Errno::ECONNREFUSED, SocketError
+          return nil
+        end
+      end
+
+      # sleep target workers
+      # @param options @see {#request_to_target_hosts}
+      def sleep_worker(options = {})
+        return request_to_target_hosts(options){|h,p| put_worker_status(h,p,Patriot::Worker::Status::SLEEP)}
+      end
+
+      # wake up target workers
+      # @param options @see {#request_to_target_hosts}
+      def wake_worker(options = {})
+        return request_to_target_hosts(options){|h,p| put_worker_status(h,p,Patriot::Worker::Status::ACTIVE)}
+      end
+
+      # change state of a worker
+      # @param host [String] host name of the target host
+      # @param port [String] port number of the worker process on the target host
+      def put_worker_status(host, port, new_status)
+        return RestClient.put("http://#{host}:#{port}/worker", :status => new_status)
+      end
+
+      # start target workers
+      # @param options @see {#request_to_target_hosts}
+      def start_worker(options = {})
+        return request_to_target_hosts(options){|h,p| controll_worker_at(h,'start')}
+      end
+
+      # stop target workers
+      # @param options @see {#request_to_target_hosts}
+      def stop_worker(options = {})
+        return request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')}
+      end
+
+      # restart target workers
+      # @param options @see {#request_to_target_hosts}
+      def restart_worker(options = {})
+        options = {:interval => 60}.merge(options)
+        target_nodes = request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')} 
+        target_nodes.keys.each{|host| target_nodes[host] = true}
+
+        port = options.has_key?(:port) ? options[:port] : @default_port
+        while(target_nodes.has_value?(true))
+          target_nodes.keys.each do |host|
+            next unless target_nodes[host] # skip already started 
+            res = get_worker_status(host,port)
+            if res.nil?
+              controll_worker_at(host,'start')
+              target_nodes[host] = false
+            else
+              if res.code == 200
+                @logger.info "status code from #{host} : #{res.code}"
+              else
+                @logger.warn "status code from #{host} : #{res.code}"
+              end
+            end
+          end
+          sleep options[:interval] if target_nodes.has_value?(true)
+        end
+      end
+
+      # execute a worker command at a remote host
+      # @param host [String] host name of the target host
+      def controll_worker_at(host, cmd)
+        ssh_cmd = "ssh -l #{@user} #{host} sudo #{WORKER_COMMAND} #{cmd}"
+        @logger.info ssh_cmd
+        puts `#{ssh_cmd}`
+      end
+
+      # upgrade libraries for target workers
+      # @param options @see {#request_to_target_hosts}
+      def upgrade_worker(options = {})
+        return request_to_target_hosts(options){|h,p| do_upgrade_at(h)}
+      end
+
+      # execute upgrade commands at a remote host
+      # @param host [String] host name of the target host
+      def do_upgrade_at(host)
+        ssh_cmd = "ssh -l #{@user} #{host} sudo #{UPGRADE_COMMAND}"
+        @logger.info ssh_cmd
+        puts `#{ssh_cmd}`
+      end
+
+    end
+  end
+end

data/lib/patriot/job_store.rb

@@ -0,0 +1,66 @@
+module Patriot
+  # namespace for JobStore
+  module JobStore
+    # state of jobs in JobStore
+    module JobState
+      # unneeded and discarded (i.e, trash)
+      DISCARDED = -2
+      # initiating
+      INIT      = -1
+      # successfully finished
+      SUCCEEDED = 0
+      # waiting to be executed
+      WAIT      = 1
+      # running currently
+      RUNNING   = 2
+      # suspended and not to be executed
+      SUSPEND   = 3
+      # executed but failed
+      FAILED    = 4
+
+      # get name of the state
+      # @param [Patriot::JobStore::JobState] state
+      # @return[String]
+      def name_of(state)
+        state = state.to_i
+        return case state
+          when -2 then "DISCARDED"
+          when -1 then "INIT"
+          when 0  then "SUCCEEDED"
+          when 1  then "WAIT"
+          when 2  then "RUNNING"
+          when 3  then "SUSPEND"
+          when 4  then "FAILED"
+          else raise "unknown state #{state}"
+        end
+      end
+      module_function :name_of
+    end
+
+    # mapping from exit code to job state
+    # @see Patriot::Command::ExitCode
+    EXIT_CODE_TO_STATE = {
+      Patriot::Command::ExitCode::SUCCEEDED       => Patriot::JobStore::JobState::SUCCEEDED,
+      Patriot::Command::ExitCode::FAILURE_SKIPPED => Patriot::JobStore::JobState::SUCCEEDED,
+      Patriot::Command::ExitCode::FAILED          => Patriot::JobStore::JobState::FAILED
+    }
+
+    # a prefix for configuration key
+    CONFIG_PREFIX = "jobstore"
+    # root (default) store_id
+    ROOT_STORE_ID = "root"
+
+    # the job identifier for initiator (always succeeded)
+    # jobs without any required products to be configured to depend on the initiator
+    INITIATOR_JOB_ID = "INITIATOR"
+    # default job priority
+    DEFAULT_PRIORITY=1 
+
+    require 'patriot/job_store/job_ticket'
+    require 'patriot/job_store/job'
+    require 'patriot/job_store/base'
+    require 'patriot/job_store/factory'
+    require 'patriot/job_store/in_memory_store'
+    require 'patriot/job_store/rdb_job_store'
+  end
+end