patriot-workflow-scheduler 0.7.2 → 0.8.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGE3Nzk3MmFkMDI1YTQxZWU4OTAxMjU3MmNlMDM0NTU5M2E2MzI4Zg==
+    YjNjNzA3NWI2YWU2ZmQ0OWZhM2E1ZmJjNTVlNTBmN2Q0YzkyYjQ3OQ==
   data.tar.gz: !binary |-
-    MzQzZTkxM2UzODU3NmIyODFjZWFhZjJiOWRjZTk1OWI2NTJiY2ZjNg==
+    MDRmNGNkOWE0MDAzYmY2NDA5MGZkYzJhNDc2N2Y0ZWM4ZjEzMGM2OQ==
 SHA512:
   metadata.gz: !binary |-
-    YTY0M2ZlZGVjZDg5YmQ1M2UzOTcxNTNkMGQ0YzA5OTMxOGQwYjU1YzliYmEz
-    MGY0MGFiNDk1NDNlOGRiZDM5MWIzZDk2ODJkYzBlODdiZWU0ZTZiNTU1Mjk4
-    NzcyMTZmZGRmZWVjNzMyN2ZmNmQyOWJjN2ZjZGFjNmQzMmNkNTU=
+    ZTY5YzAwNjFmYjNlNjllODliNGM4MzRjMjBhYzgwMDZhZWZlZjcyYWE2MmM4
+    ZGYwNDgyNTljOTg5ZTY0MDBmN2M4ZWYyOWQ4YTE5YzkwM2Y2YmE4NGU3NWI5
+    MDdkNDIwMWFmNWYzZjY3MzIzZmMzNmQ0OWQ2MTc2NWRkNWJmZDM=
   data.tar.gz: !binary |-
-    YThmNjgyYzhjNjk5YzRjZmM3NjEyYTRmMWM4NWRlOWRiZDA3MzE4MmQ1MzI1
-    ZTk2ZDY3M2E0YmM4ZmNlMDRkNGZmMGNlMDcyZDUyYWY3MjY4Mjc1YjczYTMw
-    Njk5ZTVlMzI2M2NjMDBlM2ExM2ZiZTg0NzkzY2ZkNTY2NGY5NTk=
+    NWE5OGY0NDQ2NGY4NjAwNTJmMjIwNGJiMWU5OGExMjY4ZjkxMTQyOGU2MjYy
+    Mjg5YzgxZDVhOTZlZGQzYTg1YWU3ZDYyY2Y4N2FiYzk4NmY1ODQ0MDdlYmVm
+    MWU0MWE1ZTNmY2UyYjE0NWM5Y2M0MWVhMjRjOGZiMzQ2YTFhOTI=

data/lib/patriot.rb

@@ -2,6 +2,7 @@
 require 'rubygems'
 require 'active_support'
 
+require 'patriot/version'
 require 'patriot/util'
 require 'patriot/command'
 require 'patriot/job_store'

data/lib/patriot/command.rb

@@ -1,28 +1,28 @@
 # the root name space for this scheduler
 module Patriot
   # a name space for commands
-  module Command 
+  module Command
     # a parameter key for command class name (used in jobs)
-    COMMAND_CLASS_KEY   = "COMMAND_CLASS"
+    COMMAND_CLASS_KEY   = :COMMAND_CLASS
 
     # attribute name for required products
-    REQUISITES_ATTR     = "requisites"
+    REQUISITES_ATTR     = :requisites
     # attribute name for produced products
-    PRODUCTS_ATTR       = "products"
+    PRODUCTS_ATTR       = :products
     # attribute name for job state
-    STATE_ATTR          = "state"
+    STATE_ATTR          = :state
     # attribute name for job priority
-    PRIORITY_ATTR       = "priority"
+    PRIORITY_ATTR       = :priority
     # attribute name for job execution node constraint
-    EXEC_NODE_ATTR      = "exec_node"
+    EXEC_NODE_ATTR      = :exec_node
     # attribute name for job execution host constraint
-    EXEC_HOST_ATTR      = "exec_host"
+    EXEC_HOST_ATTR      = :exec_host
     # attribute name for start time constraint
-    START_DATETIME_ATTR = "start_datetime"
+    START_DATETIME_ATTR = :start_datetime
     # attribute name for retry configration
-    POST_PROCESSORS_ATTR = "post_processors"
+    POST_PROCESSORS_ATTR = :post_processors
 
-    # a list of comman attributes
+    # a list of command attributes
     COMMON_ATTRIBUTES = [
       REQUISITES_ATTR,
       PRODUCTS_ATTR,

data/lib/patriot/command/base.rb

@@ -16,7 +16,7 @@ module Patriot
       attr_accessor :config, :parser, :test_mode, :target_datetime, :post_processors
       attr_writer   :start_datetime
 
-      # comman attributes handled distinctively (only effective in top level commands)
+      # command attributes handled distinctively (only effective in top level commands)
       volatile_attr :requisites,
                     :products,
                     :priority,
@@ -145,7 +145,7 @@ module Patriot
       def init_param
         # set parameter value to instance variable
         @param.each do |k,v|
-          raise "a reserved word #{k} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == k
+          raise "a reserved word #{k} is used as parameter name" if Patriot::Command::COMMAND_CLASS_KEY == k.to_sym
           raise "#{k} is already used in #{self.job_id}" unless instance_variable_get("@#{k}".to_sym).nil?
           # don't evaluate here since all parameters are not set to instance variables
           instance_variable_set("@#{k}".to_sym,v)
@@ -177,7 +177,7 @@ module Patriot
           val = self.instance_variable_get("@#{attr}".to_sym)
           logics.each do |l|
             unless l.call(self, attr, val)
-              raise "validation error : #{attr}=#{val} (#{self.class})" 
+              raise "validation error : #{attr}=#{val} (#{self.class})"
             end
           end
         end

data/lib/patriot/command/composite.rb

@@ -1,11 +1,11 @@
 module Patriot
   module Command
     # a command which is composed of multiple sub commands
-    class CompositeCommand < Patriot::Command::CommandGroup 
-      declare_command_name :composite_command 
-      declare_command_name :composite_job 
+    class CompositeCommand < Patriot::Command::CommandGroup
+      declare_command_name :composite_command
+      declare_command_name :composite_job
       private_command_attr :contained_commands => []
-      volatile_attr :name, :name_suffix
+      command_attr :name, :name_suffix
 
       # @return [String] the identifier of this composite command
       # @see Patriot::Command::Base#job_id
@@ -27,7 +27,9 @@ module Patriot
         @name_suffix ||= _date_
         # don't do flatten to handle nested composite commands
         @subcommands.map do |cmd|
-           cmd.build(@param).each do |cmd|
+          cmd = cmd.clone
+          cmd.build(@param).each do |cmd|
+            _validate_command(cmd)
             require cmd['requisites']
             produce cmd['products']
             @contained_commands << cmd
@@ -42,7 +44,19 @@ module Patriot
         @contained_commands.each do |c|
           c.execute
         end
-      end 
+      end
+
+      # @private
+      # validate command
+      # @param [Patriot::Command::Base] cmd
+      def _validate_command(cmd)
+        if !cmd['post_processors'].nil?
+          raise 'you cannot set "post_processor" at subcommand of composite_job\'s ' \
+            + "\n" + 'name: ' + cmd['name'] \
+            + "\n" + 'command: ' + cmd['commands'].to_s
+        end
+      end
+      private :_validate_command
 
     end
   end

data/lib/patriot/command/post_processor.rb

@@ -1,9 +1,9 @@
 # the root name space for this scheduler
 module Patriot
   # a name space for commands
-  module Command 
+  module Command
     module PostProcessor
-      POST_PROCESSOR_CLASS_KEY   = "POST_PROCESSOR_CLASS"
+      POST_PROCESSOR_CLASS_KEY   = :POST_PROCESSOR_CLASS
       require 'patriot/command/post_processor/base'
       require 'patriot/command/post_processor/skip_on_fail'
       require 'patriot/command/post_processor/retrial'

data/lib/patriot/command/post_processor/base.rb

@@ -22,8 +22,9 @@ module Patriot
 
         # @param props [Hash] properties of this post processor
         def initialize(props = {})
-          validate_props(props)
-          @props = props
+          @props = {}
+          props.each{|k,v| @props[k.to_sym] = v}
+          validate_props(@props)
         end
 
         def validate_props(props)

data/lib/patriot/command/post_processor/mail_notification.rb

@@ -4,10 +4,10 @@ module Patriot
     module PostProcessor
       class MailNotification  < Patriot::Command::PostProcessor::Base
 
-        TO_PROP_KEY = 'to'
-        ON_PROP_KEY = 'on'
+        TO_PROP_KEY = :to
+        ON_PROP_KEY = :on
 
-        declare_post_processor_name :mail_notification 
+        declare_post_processor_name :mail_notification
 
         def validate_props(props)
           raise "#{TO_PROP_KEY} is not specified" unless props.has_key?(TO_PROP_KEY)

data/lib/patriot/command/post_processor/retrial.rb

@@ -3,8 +3,8 @@ module Patriot
     module PostProcessor
       class Retrial < Patriot::Command::PostProcessor::Base
 
-        COUNT_PROP_KEY    = 'count'
-        INTERVAL_PROP_KEY = 'interval'
+        COUNT_PROP_KEY    = :count
+        INTERVAL_PROP_KEY = :interval
 
         declare_post_processor_name :retrial
 
@@ -21,7 +21,7 @@ module Patriot
             found = true
             # count first attempt in
             pp.props[COUNT_PROP_KEY] = pp.props[COUNT_PROP_KEY] - 1
-            return if pp.props[COUNT_PROP_KEY] == 0 
+            return if pp.props[COUNT_PROP_KEY] == 0
             cmd.start_datetime = Time.now + pp.props[INTERVAL_PROP_KEY]
           end
           job = cmd.to_job

data/lib/patriot/command/sh_command.rb

@@ -1,15 +1,29 @@
 module Patriot
   module Command
     # a command which executes shell scripts
+    #
+    # == Example pbc
+    #   sh {
+    #     name "test"
+    #     commands "echo '#{_date_}' > /tmp/test.out"
+    #   }
     class ShCommand < Patriot::Command::Base
       include Patriot::Util::System
 
       declare_command_name :sh
 
       command_attr :connector => '&&'
+      # @!attribute [w] commands
+      #   [String, Array] commands to execute
       command_attr :commands do |cmd, a, v|
         cmd.commands = v.is_a?(Array)? v : [v]
       end
+      # @!attribute [w] name
+      #   [String] string to be a part of job id
+      #   @see Patriot::Command::ShCommand#job_id
+      # @!attribute [w] name_suffix
+      #   [String] suffix string to be a part of job id
+      #   @see Patriot::Command::ShCommand#job_id
       command_attr :name, :name_suffix
       validate_existence :name 
 

data/lib/patriot/controller/worker_admin_controller.rb

@@ -1,4 +1,5 @@
 require 'rest_client'
+require 'base64'
 
 module Patriot
   module Controller
@@ -18,21 +19,24 @@ module Patriot
         @config = config
         @logger = create_logger(config)
         set_default_values
+        username  = config.get(Patriot::Util::Config::USERNAME_KEY, "")
+        password  = config.get(Patriot::Util::Config::PASSWORD_KEY, "")
+        @auth = 'Basic ' + Base64.encode64("#{username}:#{password}").chomp
       end
 
       # @private
       def set_default_values
-        @default_hosts = @config.get('worker_hosts') || []
-        @default_port  = @config.get('info_server_port')
-        @user          = @config.get('admin_user')
+        @default_hosts = @config.get(WORKER_HOST_KEY) || []
+        @default_port  = @config.get(INFO_SERVER_PORT_KEY)
+        @user          = @config.get(ADMIN_USER_KEY)
       end
       private :set_default_values
 
       # execute block for each target hosts
       # @param options [Hash]
       # @option options :host  a target host
-      # @option options :hosts a comman separated value of target hosts
-      # @option options :all   set true to target all hosts in the configuration 
+      # @option options :hosts a comma separated value of target hosts
+      # @option options :all   set true to target all hosts in the configuration
       # @return [Hash] a hash from host name to the result of the block
       def request_to_target_hosts(options = {}, &blk)
         hosts = []
@@ -90,7 +94,8 @@ module Patriot
       # @param host [String] host name of the target host
       # @param port [String] port number of the worker process on the target host
       def put_worker_status(host, port, new_status)
-        return RestClient.put("http://#{host}:#{port}/worker", :status => new_status)
+        resource = RestClient::Resource.new("http://#{host}:#{port}/worker")
+        return resource.put({:status => new_status}, :Authorization => @auth )
       end
 
       # start target workers
@@ -109,13 +114,13 @@ module Patriot
       # @param options @see {#request_to_target_hosts}
       def restart_worker(options = {})
         options = {:interval => 60}.merge(options)
-        target_nodes = request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')} 
+        target_nodes = request_to_target_hosts(options){|h,p| controll_worker_at(h,'stop')}
         target_nodes.keys.each{|host| target_nodes[host] = true}
 
         port = options.has_key?(:port) ? options[:port] : @default_port
         while(target_nodes.has_value?(true))
           target_nodes.keys.each do |host|
-            next unless target_nodes[host] # skip already started 
+            next unless target_nodes[host] # skip already started
             res = get_worker_status(host,port)
             if res.nil?
               controll_worker_at(host,'start')

data/lib/patriot/job_store/base.rb

@@ -63,17 +63,18 @@ module Patriot
       end
 
       # get a job
-      # @param [String] job_id
-      # @param [Hash] opts
+      # @param job_id [String] job_id
+      # @param opts [Hash]
       # @option opts [String] :include_dependency include jobs with 1-hop dependency
       # @return [Patrio::JobStore::Job] in case of include_dependency is true,
       # jobs in dependency set to :consumers/:producers as a hash from job_id to state
       def get(job_id, opts={})
+        return nil if job_id.nil?
         job = get_job(job_id)
         return if job.nil?
         if opts[:include_dependency] == true
-          job['consumers'] = get_consumers(job[Patriot::Command::PRODUCTS_ATTR]) || {}
-          job['producers'] = get_producers(job[Patriot::Command::REQUISITES_ATTR]) ||{}
+          job[:consumers] = get_consumers(job[Patriot::Command::PRODUCTS_ATTR]) || []
+          job[:producers] = get_producers(job[Patriot::Command::REQUISITES_ATTR]) || []
         end
         return job
       end
@@ -93,7 +94,7 @@ module Patriot
       def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
         raise NotImplementedError
       end
-  
+
       # get consumers of products
       # @param [Array] products a list of product name
       # @param [Hash] opts
@@ -112,6 +113,14 @@ module Patriot
         raise NotImplementedError
       end
 
+      # get nodes and edges information to render graph
+      # @param  [String]  job_id  JOB ID
+      # @param  [Hash]    opts    options
+      # @return [Array]   [nodes, edges]
+      def get_graph(job_id, opts = {})
+        raise NotImplementedError
+      end
+
       # @param [Patriot::JobStore::JobState] state
       # @param [Hash] opts
       # @option ops [Integer] :limit a max nubmer of jobs fetched at once
@@ -124,7 +133,7 @@ module Patriot
 
       # @param [Hash] opts
       # @option [Array<Patriot::JobStore::JobState>] :ignore_states
-      # @return [Hash<Patriot::JobStore::JobState, Integer>] a hash from job state to the number of jobs in the state 
+      # @return [Hash<Patriot::JobStore::JobState, Integer>] a hash from job state to the number of jobs in the state
       def get_job_size(opts = {})
         raise NotImplementedError
       end
@@ -147,7 +156,7 @@ module Patriot
         }.compact.flatten
         consumers = get_consumers(products)
         while !consumers.empty?
-          jobs = consumers.keys.map{|jid| get_job(jid)}.compact
+          jobs = consumers.map{|job| get_job(job[:job_id])}.compact
           yield self, jobs
           products = jobs.map{|j| j[Patriot::Command::PRODUCTS_ATTR]}.compact.flatten
           consumers = get_consumers(products)

data/lib/patriot/job_store/in_memory_store.rb

@@ -29,26 +29,31 @@ module Patriot
       def register(update_id, jobs)
         jobs.each{|job| raise "#{job.job_id} is not acceptable" unless acceptable?(job) }
         @mutex.synchronize do
-          jobs.each do |job|
-            job_id        = job.job_id.to_sym
-            job.update_id = update_id
-            if @jobs.has_key?(job_id) # update
-              job[Patriot::Command::STATE_ATTR] ||= @jobs[job_id][Patriot::Command::STATE_ATTR]
-            else # insert
-              # set default state
-              job[Patriot::Command::STATE_ATTR] ||= Patriot::JobStore::JobState::INIT
-            end
-            @jobs[job_id]      = job
-            @producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] || []
-            @consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] || []
-            if job[Patriot::Command::STATE_ATTR] == Patriot::JobStore::JobState::INIT
-              _set_state(job_id, Patriot::JobStore::JobState::WAIT)
-            else
-              _set_state(job_id, job[Patriot::Command::STATE_ATTR])
-            end
-          end
+          jobs.each {|job| _upsert(update_id, job) }
+        end
+      end
+
+      def _upsert(update_id, job)
+        job_id        = job.job_id.to_sym
+        if @jobs.has_key?(job_id) # update
+          original = @jobs[job_id]
+          job[Patriot::Command::STATE_ATTR] ||= original[Patriot::Command::STATE_ATTR]
+          job.update_id = original.update_id
+        else # insert
+          job[Patriot::Command::STATE_ATTR] ||= Patriot::JobStore::JobState::INIT
+          raise "update_id id should not be nil for new jobs" if update_id.nil?
+          job.update_id = update_id
+        end
+        @jobs[job_id]      = job
+        @producers[job_id] = job[Patriot::Command::PRODUCTS_ATTR] || []
+        @consumers[job_id] = job[Patriot::Command::REQUISITES_ATTR] || []
+        if job[Patriot::Command::STATE_ATTR] == Patriot::JobStore::JobState::INIT
+          _set_state(job_id, Patriot::JobStore::JobState::WAIT)
+        else
+          _set_state(job_id, job[Patriot::Command::STATE_ATTR])
         end
       end
+      private :_upsert
 
       # @see Patriot::JobStore::Base#acceptable?
       def acceptable?(job)
@@ -141,6 +146,8 @@ module Patriot
 
       # @see Patriot::JobStore::Base#get_job
       def get_job(job_id)
+        return nil if job_id.nil?
+        raise "string is expected but job_id is a #{job_id.class}" unless job_id.is_a?(String)
         return @jobs[job_id.to_sym]
       end
 
@@ -148,26 +155,34 @@ module Patriot
       def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
         opts = {:include_attrs => []}.merge(opts)
         products = [products] unless products.is_a?(Array)
-        producers = {}
-        products.each{|product| 
-          @producers.map{|pid, prods| 
-            producers[pid.to_s] = @jobs[pid].filter_attributes(opts[:include_attrs]) if prods.include?(product)
+        producers = []
+        products.each{|product|
+          @producers.map{|pid, prods|
+            if prods.include?(product)
+              job = @jobs[pid].filter_attributes(opts[:include_attrs])
+              job[:job_id] = pid.to_s
+              producers.push(job)
+            end
           }
         }
-        return producers
+        return producers.uniq
       end
-  
+
       # @see Patriot::JobStore::Base#get_consumers
       def get_consumers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
         opts = {:include_attrs => []}.merge(opts)
         products = [products] unless products.is_a?(Array)
-        consumers = {}
+        consumers = []
         products.each{|product|
-          @consumers.map{|pid, prods| 
-            consumers[pid.to_s] = @jobs[pid].filter_attributes(opts[:include_attrs]) if prods.include?(product)
+          @consumers.map{|pid, prods|
+            if prods.include?(product)
+              job = @jobs[pid].filter_attributes(opts[:include_attrs])
+              job[:job_id] = pid.to_s
+              consumers.push(job)
+            end
           }
         }
-        return consumers
+        return consumers.uniq
       end
 
       # @see Patriot::JobStore::Base#find_jobs_by_state
@@ -193,6 +208,83 @@ module Patriot
         return @job_history[job_id.to_sym] || []
       end
 
+      # get nodes and edges information to render graph
+      # @param  [String]  job_id  JOB ID
+      # @param  [Hash]    opts    options
+      # @return [Array]   [nodes, edges]
+      def get_graph(job_id, opts = {})
+        job = get(job_id)
+        history = get_execution_history(job_id, {})[0]
+
+        hashed_job = {
+          :job_id => job.job_id,
+          :history => history,
+          :depth => 0
+        }.merge(job.attributes)
+
+        # set self node
+        nodes = {job_id => hashed_job}
+        edges = []
+
+        _set_dependency(
+          :producers,
+          opts[:producer_depth],
+          nodes,
+          edges,
+          hashed_job
+        )
+
+        _set_dependency(
+          :consumers,
+          opts[:consumer_depth],
+          nodes,
+          edges,
+          hashed_job
+        )
+
+        return {:nodes => nodes, :edges => edges}
+      end
+
+      # get dependency and set nodes and edges
+      #
+      # @private
+      # @param  [Symbol]  direction       :producers or :consumers
+      # @param  [Integer] depth           dependency depth to get
+      # @param  [Hash]    nodes           nodes to set for dager-d3
+      # @param  [Array]   edges           edges to set for dager-d3
+      # @param  [Hash]    base_job        base job to get dependency
+      def _set_dependency(direction, depth, nodes, edges, base_job)
+        return if nodes[base_job[:job_id]][:depth] == depth
+
+        base_job[direction].map{|depend_job|
+          job = get(depend_job[:job_id])
+          history = get_execution_history(depend_job[:job_id], {})[0]
+
+          hashed_job = {
+            :job_id => job.job_id,
+            :history => history,
+            :depth => base_job[:depth] + 1
+          }.merge(job.attributes)
+
+          nodes[job.job_id] = hashed_job
+          if direction == :producers
+            edges.push([job.job_id, base_job[:job_id]])
+          else
+            edges.push([base_job[:job_id], job.job_id])
+          end
+
+          # call recursively
+          _set_dependency(
+            direction,
+            depth,
+            nodes,
+            edges,
+            hashed_job
+          )
+        }
+      end
+      private :_set_dependency
+
       # @see Patriot::JobStore::Base#get_job_size
       def get_job_size(opts = {})
         opts  = {:ignore_states => []}.merge(opts)
@@ -208,7 +300,7 @@ module Patriot
       # @see Patriot::JobStore::Base#delete_job
       def delete_job(job_id)
         job_id = job_id.to_sym
-        @mutex.synchronize do 
+        @mutex.synchronize do
           @job_states.each{|s,js| js.delete(job_id)}
           @jobs.delete(job_id)
           @producers.delete(job_id)