patriot-workflow-scheduler 0.6.1

data/lib/patriot/job_store/base.rb

@@ -0,0 +1,159 @@
+require 'patriot/util'
+
+module Patriot
+  module JobStore
+    # base class of JobStore
+    class Base
+      include Patriot::Util::Logger
+
+      # @param [String] store_id identifier of this store
+      # @param [Patriot::Util::Config::Base] config configuration of this store
+      def initialize(store_id, config)
+        raise NotImplementedError
+      end
+
+      # register the given jobs with the given update_id
+      # @param [Integer] update_id
+      # @param [Array] jobs a list of jobs to be registered
+      def register(update_id, jobs)
+        raise NotImplementedError
+      end
+
+      # check whether the command can be stored as a job to this job_store
+      # @param [Patriot::Command::Base] command
+      # @return [Boolean] true if the command can be converted to a job
+      def acceptable?(command)
+        raise NotImplementedError
+      end
+
+      # get job tickets for jobs which are ready to execute
+      # @param [String] host the host name of the client
+      # @param [Array] nodes array of nodes on the client
+      # @param opts [Hash]
+      # @option opts [Integer] :fetch_limit the max number of tickets
+      def get_job_tickets(host, nodes, opts = {})
+        raise NotImplementedError
+      end
+
+      # offer to execute a job specified with a job_ticket
+      # If the job is ready to execute, the state of job is set to {Patriot::JobStore::JobState::RUNNING}.
+      # A response of this method is a Hash including
+      # * :execution_id the identifier of the execution (used to identify history record)
+      # * :command an instance of command for the offered job
+      # @param [Patriot::JobStore::JobTicket] job_ticket the ticket of the job of which execution is offered.
+      # @return [Hash] response for the offer
+      def offer_to_execute(job_ticket)
+        raise NotImplementedError
+      end
+
+      # report completion status a job specified with a job_ticket
+      # The state of the job should be changed according to the completion status.
+      # @param [Patriot::JobStore::JobTicket] job_ticket the ticket of the job of which completion is reported
+      # @return [Boolean] true if the job exists and the state is updated, otherwise false
+      def report_completion_status(job_ticket)
+        raise NotImplementedError
+      end
+
+      # set jobs state
+      # @param [Integer] update_id
+      # @param [Array] job_ids list of job_ids
+      # @param [Integer] new_state new state of the job.
+      def set_state(update_id, job_ids, new_state)
+        raise NotImplementedError
+      end
+
+      # get a job
+      # @param [String] job_id
+      # @param [Hash] opts
+      # @option opts [String] :include_dependency include jobs with 1-hop dependency
+      # @return [Patrio::JobStore::Job] in case of include_dependency is true,
+      # jobs in dependency set to :consumers/:producers as a hash from job_id to state
+      def get(job_id, opts={})
+        job = get_job(job_id)
+        return if job.nil?
+        if opts[:include_dependency] == true
+          job['consumers'] = get_consumers(job[Patriot::Command::PRODUCTS_ATTR]) || {}
+          job['producers'] = get_producers(job[Patriot::Command::REQUISITES_ATTR]) ||{}
+        end
+        return job
+      end
+
+      # get a job data
+      # @param [String] job_id
+      # @return [Patriot::JobStore::Job]
+      def get_job(job_id)
+        raise NotImplementedError
+      end
+
+      # get producers of products
+      # @param [Array] products a list of product name
+      # @param [Hash] opts
+      # @option opt [Array] :include_attrs a list of attribute included in results
+      # @return [Hash] a hash from job_id to its state
+      def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
+        raise NotImplementedError
+      end
+  
+      # get consumers of products
+      # @param [Array] products a list of product name
+      # @param [Hash] opts
+      # @option opt [Array] :include_attrs a list of attribute included in results
+      # @return [Hash] a hash from job_id to its state
+      def get_consumers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
+        raise NotImplementedError
+      end
+
+      # get execution histories of the specified job
+      # @param [String] job_id
+      # @param [Hash] opts
+      # @option opts [Integer] :limit a max number of history records (default 1)
+      # @option opts [Symbol] :order order of record [:DESC or :ASC, default is :DESC]
+      def get_execution_history(job_id, opts = {})
+        raise NotImplementedError
+      end
+
+      # @param [Patriot::JobStore::JobState] state
+      # @param [Hash] opts
+      # @option ops [Integer] :limit a max nubmer of jobs fetched at once
+      # @option ops [Integer] :offset the number of records skipped before beginning to include to a result
+      # @option ops [String] :filter_exp additional condition on job_id in a LIKE expression
+      # @return [Array] an array of job_id  which is in the given state
+      def find_jobs_by_state(state, opts = {})
+        raise NotImplementedError
+      end
+
+      # @param [Hash] opts
+      # @option [Array<Patriot::JobStore::JobState>] :ignore_states
+      # @return [Hash<Patriot::JobStore::JobState, Integer>] a hash from job state to the number of jobs in the state 
+      def get_job_size(opts = {})
+        raise NotImplementedError
+      end
+
+      # delete the job from this job_store
+      # @param [String] job_id
+      def delete_job(job_id)
+        raise NotImplementedError
+      end
+
+      # Process subsequent jobs with a given block.
+      # The block is called for each dependency depth.
+      # @param job_ids [Array<String>]
+      # @yieldparam job_store [Patriot::JobStore::Base] this job_store
+      # @yieldparam jobs [Patriot::JobStore::Job] subsequet jobs
+      def process_subsequent(job_ids, &blk)
+        products = job_ids.map{|jid|
+          job = get_job(jid)
+          job.nil? ? nil : job[Patriot::Command::PRODUCTS_ATTR]
+        }.compact.flatten
+        consumers = get_consumers(products)
+        while !consumers.empty?
+          jobs = consumers.keys.map{|jid| get_job(jid)}.compact
+          yield self, jobs
+          products = jobs.map{|j| j[Patriot::Command::PRODUCTS_ATTR]}.compact.flatten
+          consumers = get_consumers(products)
+        end
+      end
+
+    end
+  end
+end

data/lib/patriot/job_store/factory.rb

@@ -0,0 +1,19 @@
+module Patriot
+  module JobStore
+    # a moulde for a factory method of JobStores
+    module Factory
+      # create JobStore for given store_id based on the configuration
+      # @param store_id [String] JobStore ID to identify configuration parameters
+      # @param config [Patriot::Util::Config::Base] configuration to create a JobStore
+      # @return [Patriot::JobStore::Base]
+      def create_jobstore(store_id, config)
+        cls = config.get([Patriot::JobStore::CONFIG_PREFIX, store_id, "class"].join("."))
+        # TODO set default store
+        raise "class for job store #{store_id} is not specified" if cls.nil?
+        job_store = eval(cls).new(store_id, config)
+        return job_store
+      end
+      module_function :create_jobstore
+    end
+  end
+end

data/lib/patriot/job_store/in_memory_store.rb

@@ -0,0 +1,252 @@
+require 'thread'
+
+module Patriot
+  module JobStore
+    # a JobStore implementation on memory
+    class InMemoryStore < Patriot::JobStore::Base
+      include Patriot::Util::Logger
+      include Patriot::Util::Retry
+
+      # @see Patriot::JobStore::Base#initialize
+      def initialize(store_id, config)
+        @config = config
+        @logger = create_logger(config)
+        @mutex  = Mutex.new
+        @jobs       = {} # hash from job_id to job content in hash
+        # hash from state to list of job_id
+        @job_states = {Patriot::JobStore::JobState::INIT      => [],
+                       Patriot::JobStore::JobState::SUCCEEDED => [Patriot::JobStore::INITIATOR_JOB_ID],
+                       Patriot::JobStore::JobState::WAIT      => [],
+                       Patriot::JobStore::JobState::RUNNING   => [],
+                       Patriot::JobStore::JobState::SUSPEND   => [],
+                       Patriot::JobStore::JobState::FAILED    => []}
+        @producers   = {} # hash from job_id to produces products
+        @consumers   = {} # hash from job_id to referece products
+        @job_history = {} # hash from job_id to a array of its execution hisotry
+      end
+
+      # @see Patriot::JobStore::Base#register
+      def register(update_id, jobs)
+        jobs.each{|job| raise "#{job.job_id} is not acceptable" unless acceptable?(job) }
+        @mutex.synchronize do
+          jobs.each do |job|
+            job_id        = job.job_id.to_sym
+            job.update_id = update_id
+            if @jobs.has_key?(job_id) # update
+              job[Patriot::Command::STATE_ATTR] ||= @jobs[job_id][Patriot::Command::STATE_ATTR]
+            else # insert
+              # set default state
+              job[Patriot::Command::STATE_ATTR] ||= Patriot::JobStore::JobState::INIT
+            end
+            @jobs[job_id]      = job
+            @producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] unless job[Patriot::Command::PRODUCTS_ATTR].nil?
+            @consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] unless job[Patriot::Command::REQUISITES_ATTR].nil?
+            if job[Patriot::Command::STATE_ATTR] == Patriot::JobStore::JobState::INIT
+              _set_state(job_id, Patriot::JobStore::JobState::WAIT)
+            else
+              _set_state(job_id, job[Patriot::Command::STATE_ATTR])
+            end
+          end
+        end
+      end
+
+      # @see Patriot::JobStore::Base#acceptable?
+      def acceptable?(job)
+        raise "invalid class #{job.class}" unless job.is_a?(Patriot::JobStore::Job)
+        return true
+      end
+
+      # @see Patriot::JobStore::Base#get_job_tickets
+      def get_job_tickets(host, nodes, options = {})
+        nodes = [nodes] unless nodes.is_a?(Array)
+        @mutex.synchronize do
+          return @job_states[Patriot::JobStore::JobState::WAIT].map do |wid|
+            job = @jobs[wid]
+            # check host and node
+            next unless job[Patriot::Command::EXEC_NODE_ATTR].nil?      || nodes.include?(job[Patriot::Command::EXEC_NODE_ATTR])
+            next unless job[Patriot::Command::EXEC_HOST_ATTR].nil?      || host == job[Patriot::Command::EXEC_HOST_ATTR]
+            next unless job[Patriot::Command::START_DATETIME_ATTR].nil? || Time.now > job[Patriot::Command::START_DATETIME_ATTR]
+            # check dependency
+            reference = @consumers[wid] || []
+            producers = @producers.map{|pid, prods| pid unless (prods & reference).empty?}.compact
+            next if !reference.empty? && producers.empty? # no producer exists
+            next if producers.any?{|pjid| !@job_states[Patriot::JobStore::JobState::SUCCEEDED].include?(pjid)}
+            JobTicket.new(wid.to_s, job.update_id, job[Patriot::Command::EXEC_NODE_ATTR])
+          end.compact
+        end
+      end
+
+      # @see Patriot::JobStore::Base#offer_to_execute
+      def offer_to_execute(job_ticket)
+        job_id    = job_ticket.job_id.to_sym
+        update_id = job_ticket.update_id
+        @mutex.synchronize do
+          unless _check_and_set_state(job_id, update_id, Patriot::JobStore::JobState::WAIT, Patriot::JobStore::JobState::RUNNING)
+            @logger.debug("execution of job: #{job_id} is skipped")
+            return
+          end
+          job = @jobs[job_id]
+          raise "no entry found for #{job_ticket}" if job.nil?
+          begin
+            # TODO make the max number of histories configurable and keep multiple histories
+            execution_id         = Time.now.to_i
+            @job_history[job_id] = [{:id       => execution_id,
+                                     :job_id   => job_id.to_s,
+                                     :host     => job_ticket.exec_host,
+                                     :node     => job_ticket.exec_node,
+                                     :thread   => job_ticket.exec_thread,
+                                     :begin_at => Time.now
+                                   }]
+            return {:execution_id => execution_id, :command => job.to_command(@config)}
+          rescue Exception => e
+            _check_and_set_state(job_id, update_id, Patriot::JobStore::JobState::RUNNING, Patriot::JobStore::JobState::FAILED)
+            raise e
+          end
+        end
+      end
+
+      # @see Patriot::JobStore::Base#report_completion_status
+      def report_completion_status(job_ticket)
+        job_id    = job_ticket.job_id.to_sym
+        update_id = job_ticket.update_id
+        exit_code = job_ticket.exit_code
+        raise "exit code is not set " if exit_code.nil?
+        state     = Patriot::JobStore::EXIT_CODE_TO_STATE[exit_code]
+        raise "invalid exit code #{exit_code} " if state.nil?
+        @mutex.synchronize do
+          # TODO save finish_time to history server
+          last_history = @job_history[job_id]
+          raise "illegal state job_history is not set for #{job_id}" if last_history.nil? || last_history.empty?
+          last_history = last_history[0]
+          # TODO make the max number of histories configurable and keep multiple histories
+          @job_history[job_id] = [last_history.merge({:exit_code => exit_code, :end_at => Time.now, :description => job_ticket.description})]
+          return _check_and_set_state(job_id, update_id, Patriot::JobStore::JobState::RUNNING, state)
+        end
+      end
+
+      # @see Patriot::JobStore::Base#set_state
+      def set_state(update_id, job_ids, new_state)
+        @mutex.synchronize do
+          job_ids = job_ids.map do |jid|
+            @jobs[jid.to_sym][Patriot::Command::STATE_ATTR] = new_state
+            jid.to_sym
+          end
+          @job_states.each do |s,jobs|
+            next if s == new_state
+            @job_states[s] -= job_ids
+          end
+          @job_states[new_state] += job_ids
+        end
+      end
+
+      # @see Patriot::JobStore::Base#get_job
+      def get_job(job_id)
+        return @jobs[job_id.to_sym]
+      end
+
+      # @see Patriot::JobStore::Base#get_producers
+      def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
+        opts = {:include_attrs => []}.merge(opts)
+        products = [products] unless products.is_a?(Array)
+        producers = {}
+        products.each{|product| 
+          @producers.map{|pid, prods| 
+            producers[pid.to_s] = @jobs[pid].filter_attributes(opts[:include_attrs]) if prods.include?(product)
+          }
+        }
+        return producers
+      end
+  
+      # @see Patriot::JobStore::Base#get_consumers
+      def get_consumers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
+        opts = {:include_attrs => []}.merge(opts)
+        products = [products] unless products.is_a?(Array)
+        consumers = {}
+        products.each{|product|
+          @consumers.map{|pid, prods| 
+            consumers[pid.to_s] = @jobs[pid].filter_attributes(opts[:include_attrs]) if prods.include?(product)
+          }
+        }
+        return consumers
+      end
+
+      # @see Patriot::JobStore::Base#find_jobs_by_state
+      def find_jobs_by_state(state, opts = {})
+        all_records = @job_states[state] - [Patriot::JobStore::INITIATOR_JOB_ID]
+        size        = all_records.size
+        opts        = {:limit => size, :offset => 0}.merge(opts)
+        filter      = opts.has_key?(:filter_exp) ? Regexp.new(opts[:filter_exp].gsub(/(?<!\\)%/,'.*').gsub(/(?<!\\)_/,'.')) : nil
+        result      = []
+        opts[:offset].upto(size).each do |i|
+          break if i >= size
+          break if result.size >= opts[:limit]
+          job_id = all_records[size - 1 - i].to_s
+          next  if !filter.nil? && !filter.match(job_id)
+          result << job_id
+        end
+        return result
+      end
+
+      # @see Patriot::JobStore::Base#get_execution_history
+      def get_execution_history(job_id, opts = {})
+        opts = {:limit => 1, :order => :DESC}
+        return @job_history[job_id.to_sym] || []
+      end
+
+      # @see Patriot::JobStore::Base#get_job_size
+      def get_job_size(opts = {})
+        opts  = {:ignore_states => []}.merge(opts)
+        sizes = {}
+        @job_states.each do |s,js|
+          next if opts[:ignore_states].include?(s)
+          sizes[s] = js.size
+          sizes[s] = sizes[s] -1 if s == Patriot::JobStore::JobState::SUCCEEDED
+        end
+        return sizes
+      end
+
+      # @see Patriot::JobStore::Base#delete_job
+      def delete_job(job_id)
+        job_id = job_id.to_sym
+        @mutex.synchronize do 
+          @job_states.each{|s,js| js.delete(job_id)}
+          @jobs.delete(job_id)
+          @producers.delete(job_id)
+          @consumers.delete(job_id)
+        end
+      end
+
+      ### private
+
+      # not thread safe. should be locked around invocation
+      # @param [Symbol] job_id
+      # @param [Integer] update_id
+      # @param [Integer] prev_state
+      # @param [Integer] post_state
+      # @return [Boolean] true if state is changed, otherwise false
+      def _check_and_set_state(job_id, update_id, prev_state, post_state)
+        return false unless @job_states[prev_state].include?(job_id)
+        return false unless @jobs[job_id].update_id == update_id
+        _set_state(job_id, post_state)
+        return true
+      end
+      private :_check_and_set_state
+
+      # set job state
+      # @param [String] job_id
+      # @param [Integer] new_state new state of the job. set nil to keep_state
+      def _set_state(job_id, new_state)
+        return if new_state.nil?
+        job_id = job_id.to_sym
+        @job_states.each do |s,jobs|
+          deleted_id = jobs.delete(job_id)
+          break unless deleted_id.nil?
+        end
+        @jobs[job_id][Patriot::Command::STATE_ATTR] = new_state
+        @job_states[new_state] << job_id
+      end
+      private :_set_state
+
+    end
+  end
+end

data/lib/patriot/job_store/job.rb

@@ -0,0 +1,118 @@
+module Patriot
+  module JobStore
+    # a record stored in jobstore
+    class Job
+
+      attr_accessor :job_id, :update_id, :attributes
+
+      # @param job_id [String]  the identifier of the job
+      def initialize(job_id)
+        @job_id     = job_id
+        @attributes = {
+                        Patriot::Command::PRIORITY_ATTR => Patriot::JobStore::DEFAULT_PRIORITY,
+                        Patriot::Command::STATE_ATTR    => Patriot::JobStore::JobState::INIT
+                      }
+      end
+
+      # set an attribute to this job
+      # @param k [String] attribute name
+      # @param v [Object] attribute value
+      def []=(k,v)
+        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        @attributes[k] = v
+      end
+
+      # get an attribute to this job
+      # @param k [String] attribute name
+      # @return [Object] the attribute value
+      def [](k)
+        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        return @attributes[k]
+      end
+
+      # delete an attribute
+      # @param k [String] attribute name
+      # @return [Object] the deleted attribute value
+      def delete(k)
+        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        return @attributes.delete(k)
+      end
+
+      # read the content of command
+      # @param command [Patriot::Command::Base] a command loaded to this job
+      def read_command(command)
+        Patriot::Command::COMMON_ATTRIBUTES.each do |attr|
+          value = command.instance_variable_get("@#{attr}".to_sym)
+          self[attr] = value unless value.nil?
+        end
+        _to_stdobj(command).each{|k,v| self[k] = v}
+      end
+
+      # @private
+      # convert a given object to an object only includes standand objects can be converted to JSON.
+      # in other words, convert Command instances in the object to hash
+      def _to_stdobj(obj)
+        if obj.is_a?(Patriot::Command::Base)
+          hash = {}
+          hash[Patriot::Command::COMMAND_CLASS_KEY] = obj.class.to_s.gsub(/::/, '.')
+          obj.class.serde_attrs.each do |attr|
+            value = obj.instance_variable_get("@#{attr}".to_sym)
+            hash[attr.to_s] = _to_stdobj(value) unless value.nil?
+          end
+          return hash
+        elsif obj.is_a?(Hash)
+          hash = {}
+          obj.each{|k,v| hash[k.to_s] = _to_stdobj(v)}
+          return hash
+        elsif obj.is_a?(Array)
+          return obj.map{|e| _to_stdobj(e)}
+        else
+          return obj
+        end
+      end
+      private :_to_stdobj
+
+      # @param config [Patriot::Util::Command::Base] configuration for building a command
+      # @return [Patriot::Command::Base] an executable for this job
+      def to_command(config)
+        raise "configuration is not set" if config.nil?
+        return _from_stdobj(self.attributes, config)
+      end
+
+      # @private
+      # convert corresponding objects in the given argument into Command instances.
+      # @param obj [Object] a object to be deserialized
+      # @param config [Patriot::util::Config::Base] configuration used for deserialization
+      # @return [Object] a starndard object (primitive or command, or array) for the obj
+      def _from_stdobj(obj, config)
+        if obj.is_a?(Hash)
+          if obj.has_key?(Patriot::Command::COMMAND_CLASS_KEY)
+            cmd_cls = obj.delete(Patriot::Command::COMMAND_CLASS_KEY)
+            cmd_cls = cmd_cls.split('.').inject(Object){|c,name| c.const_get(name)}
+            cmd     = cmd_cls.new(config)
+            obj.each do |k,v|
+              cmd.instance_variable_set("@#{k}".to_sym, _from_stdobj(v, config))
+            end
+            return cmd
+          else
+            hash = {}
+            obj.each{|k,v| hash[k] = _from_stdobj(v, config)}
+            return hash
+          end
+        elsif obj.is_a?(Array)
+          return obj.map{|e| _from_stdobj(e, config)}
+        else
+          return obj
+        end
+      end
+
+      # @param attrs [Array<String>] a list of attribute names
+      # @return [Hash] a set of attribute name value pairs for specified attributes
+      def filter_attributes(attrs)
+        filtered = {}
+        attrs.each{|a| filtered[a] = self[a]}
+        return filtered
+      end
+    end
+  end
+end