patriot-workflow-scheduler 0.7.2 → 0.8.0

data/lib/patriot/job_store/job.rb

@@ -18,7 +18,7 @@ module Patriot
       # @param k [String] attribute name
       # @param v [Object] attribute value
       def []=(k,v)
-        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        raise "key #{k} should be symbol but #{k.class}" unless k.is_a?(Symbol)
         @attributes[k] = v
       end
 
@@ -26,7 +26,7 @@ module Patriot
       # @param k [String] attribute name
       # @return [Object] the attribute value
       def [](k)
-        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        raise "key #{k} should be symbol but #{k.class}" unless k.is_a?(Symbol)
         return @attributes[k]
       end
 
@@ -34,7 +34,7 @@ module Patriot
       # @param k [String] attribute name
       # @return [Object] the deleted attribute value
       def delete(k)
-        raise "key #{k} should be string but #{k.class}" unless k.is_a?(String)
+        raise "key #{k} should be symbol but #{k.class}" unless k.is_a?(Symbol)
         return @attributes.delete(k)
       end
 
@@ -57,14 +57,14 @@ module Patriot
           hash[Patriot::Command::COMMAND_CLASS_KEY] = obj.class.to_s.gsub(/::/, '.')
           obj.class.serde_attrs.each do |attr|
             value = obj.instance_variable_get("@#{attr}".to_sym)
-            hash[attr.to_s] = _to_stdobj(value) unless value.nil?
+            hash[attr.to_sym] = _to_stdobj(value) unless value.nil?
           end
           return hash
         elsif obj.is_a?(Patriot::Command::PostProcessor::Base)
           hash = {}
           hash[Patriot::Command::PostProcessor::POST_PROCESSOR_CLASS_KEY] = obj.class.to_s.gsub(/::/, '.')
           obj.props.each do |k,v|
-            hash[k.to_s] = _to_stdobj(v) unless v.nil?
+            hash[k.to_sym] = _to_stdobj(v) unless v.nil?
           end
           return hash
         elsif obj.is_a?(Hash)
@@ -107,7 +107,7 @@ module Patriot
             return cmd_cls.new(obj)
           else
             hash = {}
-            obj.each{|k,v| hash[k] = _from_stdobj(v, config)}
+            obj.each{|k,v| hash[k.to_s] = _from_stdobj(v, config)}
             return hash
           end
         elsif obj.is_a?(Array)
@@ -117,7 +117,7 @@ module Patriot
         end
       end
 
-      # @param attrs [Array<String>] a list of attribute names
+      # @param attrs [Array<Symbol>] a list of attribute names
       # @return [Hash] a set of attribute name value pairs for specified attributes
       def filter_attributes(attrs)
         filtered = {}

data/lib/patriot/job_store/job_ticket.rb

@@ -13,6 +13,7 @@ module Patriot
       # @param [Integer] update_id
       # @param [String] node the name of node on which the job should be executed
       def initialize(job_id, update_id, node=nil)
+        raise "job_id should be Symbol but #{job_id} is a #{job_id.class}" unless job_id.is_a?(String)
         @job_id    = job_id
         @update_id = update_id
         @node      = node

data/lib/patriot/job_store/rdb_job_store.rb

@@ -58,11 +58,12 @@ module Patriot
 
       # @see Patriot::JobStore::Base#register
       def register(update_id, jobs)
+        jobs = [jobs] unless jobs.is_a? Array
         jobs.each{|job| raise "#{job.job_id} is not acceptable" unless acceptable?(job) }
         @logger.info "start to register jobs"
         connect(@db_config) do |c|
-          jobs.each{|job| upsert_job(update_id, job, c)}
-          c.update(JOB_TABLE, 
+          jobs.each{|job| _upsert_job(update_id, job, c)}
+          c.update(JOB_TABLE,
                    {:state => Patriot::JobStore::JobState::WAIT},
                    {:state => Patriot::JobStore::JobState::INIT, :update_id => update_id}
                   )
@@ -70,10 +71,11 @@ module Patriot
         @logger.info "job registration finished"
       end
 
-      def upsert_job(update_id, job, c)
-        new_vals = {:job_id => job.job_id, :update_id => update_id, :priority => DEFAULT_PRIORITY}
+      def _upsert_job(update_id, job, c)
+        new_vals = {:job_id => job.job_id, :priority => DEFAULT_PRIORITY}
+        new_vals[:update_id] = update_id unless update_id.nil?
         job_attr = job.attributes.dup
-        # extract and remove comman attributes
+        # extract and remove command attributes
         requisites = job_attr.delete(Patriot::Command::REQUISITES_ATTR) || []
         products   = job_attr.delete(Patriot::Command::PRODUCTS_ATTR)   || []
 
@@ -88,6 +90,7 @@ module Patriot
         new_vals[:content] = JSON.generate(job_attr)
 
         if prev_vals.empty?
+          raise "update_id should not be nil for new jobs" if new_vals[:update_id].nil?
           new_vals[:state] ||= Patriot::JobStore::JobState::INIT # set default state
           serial_id = c.insert(JOB_TABLE, new_vals)
         elsif prev_vals.size == 1
@@ -97,14 +100,14 @@ module Patriot
 
         raise "failed to upsert a job #{j}" if serial_id.nil?
 
-        update_dependency(serial_id, requisites, CONSUMER_TABLE, c)
-        update_dependency(serial_id, products,   PRODUCER_TABLE, c) 
+        _update_dependency(serial_id, requisites, CONSUMER_TABLE, c)
+        _update_dependency(serial_id, products,   PRODUCER_TABLE, c)
         # set dependency for initiator jobs
         c.insert(FLOW_TABLE, {:producer_id => @initiator_id, :consumer_id => serial_id}, {:ignore => true}) if requisites.empty?
       end
-      private :upsert_job
+      private :_upsert_job
 
-      def update_dependency(serial_id, updated_products, updated_table, conn)
+      def _update_dependency(serial_id, updated_products, updated_table, conn)
         raise "unknown dependency table #{updated_table}" unless [CONSUMER_TABLE, PRODUCER_TABLE].include?(updated_table)
         updated_col    = updated_table == CONSUMER_TABLE ? :consumer_id : :producer_id
         opposite_table = updated_table == CONSUMER_TABLE ? PRODUCER_TABLE  : CONSUMER_TABLE
@@ -126,7 +129,7 @@ module Patriot
           end
         end
       end
-      private :update_dependency
+      private :_update_dependency
 
       # @see Patriot::JobStore::Base#acceptable?
       def acceptable?(job)
@@ -143,7 +146,7 @@ module Patriot
       def get_job_tickets(host, nodes, options = {})
         nodes = [nodes] unless nodes.is_a?(Array)
         begin
-          query = generate_fetching_job_sql(host, nodes,options)
+          query = _generate_fetching_job_sql(host, nodes,options)
           @logger.debug "fetchings job by #{query}"
           connect(@db_config) do |c|
             return c.execute_statement(query).map{|r| Patriot::JobStore::JobTicket.new(r.job_id, r.update_id, r.node) }
@@ -154,25 +157,25 @@ module Patriot
         end
       end
 
-      def generate_fetching_job_sql(host, nodes, options)
+      def _generate_fetching_job_sql(host, nodes, options)
         node_condition = (nodes.map{|n| "c.node = '#{n}'" } | ["c.node IS NULL"]).join(" OR ")
         query          = <<"END_OB_QUERY"
           SELECT c.#{TICKET_COLUMNS[0]}, c.#{TICKET_COLUMNS[1]}, c.#{TICKET_COLUMNS[2]}
-          FROM flows f 
-          JOIN jobs c on c.id = f.consumer_id 
-          JOIN jobs p on f.producer_id = p.id 
+          FROM flows f
+          JOIN jobs c on c.id = f.consumer_id
+          JOIN jobs p on f.producer_id = p.id
           WHERE c.state=#{Patriot::JobStore::JobState::WAIT}
               AND (#{node_condition})
               AND (c.host = '#{host}' OR c.host IS NULL)
               AND c.content IS NOT NULL
               AND (c.start_after IS NULL  OR c.start_after < current_timestamp)
-          GROUP BY f.consumer_id HAVING Min(p.state=#{Patriot::JobStore::JobState::SUCCEEDED})=1 
+          GROUP BY f.consumer_id HAVING Min(p.state=#{Patriot::JobStore::JobState::SUCCEEDED})=1
           ORDER BY c.priority
 END_OB_QUERY
         query = "#{query} LIMIT #{options[:fetch_limit]} " if options.has_key?(:fetch_limit)
         return query.gsub(/(\r|\n|\s+)/, ' ')
       end
-      private :generate_fetching_job_sql
+      private :_generate_fetching_job_sql
 
       # @see Patriot::JobStore::Base#offer_to_execute
       def offer_to_execute(job_ticket)
@@ -190,8 +193,8 @@ END_OB_QUERY
           record = c.select(JOB_TABLE, {:job_id => job_ticket.job_id})
           raise "duplicated entry found for #{job_ticket}" if record.size > 1
           raise "no entry found for #{job_ticket}" if record.empty?
-          job = record_to_job(record[0])
-          begin 
+          job = _record_to_job(record[0])
+          begin
             return {:execution_id => execution_id, :command => job.to_command(@config)}
           rescue Exception => e
             marked = _check_and_set_state(job_ticket, Patriot::JobStore::JobState::RUNNING, Patriot::JobStore::JobState::FAILED, c)
@@ -222,7 +225,7 @@ END_OB_QUERY
           @logger.info("definition or state of job: #{job_ticket.job_id} is changed and its state is not changed")
           return false
         elsif num_updated != 1
-          raise "illegal state: #{job_ticket.job_id} has more than #{num_updated} records" 
+          raise "illegal state: #{job_ticket.job_id} has more than #{num_updated} records"
         end
         return true
       end
@@ -235,61 +238,165 @@ END_OB_QUERY
         connect(@db_config){|c| c.execute_statement(stmt, :update)}
       end
 
+      # @param  [String]  job_id  JOB ID
+      # @return [Patriot::JobStore::Job]
       # @see Patriot::JobStore::Base#get_job
       def get_job(job_id)
-        connect(@db_config) do |c|
-          records = c.select(JOB_TABLE, {:job_id => job_id})
-          return nil if records.empty?
-          raise "duplicate job_ticket for #{job_id}" unless records.size == 1
-          record = records[0]
-          serial_id = record.to_hash[:id]
-          job = record_to_job(record)
-          job[Patriot::Command::PRODUCTS_ATTR] = c.select(PRODUCER_TABLE, {:job_id => serial_id}).map{|r| r.product}
-          job[Patriot::Command::REQUISITES_ATTR] = c.select(CONSUMER_TABLE, {:job_id => serial_id}).map{|r| r.product}
-          return job
-        end
+        connect(@db_config) {|c| return _get_job(job_id, c)}
       end
 
+      # @param  [String]  job_id  JOB ID
+      # @param  [Patriot::Util::DBClient::Base] c
+      # @return [Patriot::JobStore::Job]
+      def _get_job(job_id, c)
+        records = c.select(JOB_TABLE, {:job_id => job_id})
+        return nil if records.empty?
+        raise "duplicate job_ticket for #{job_id}" unless records.size == 1
+        record = records[0]
+        serial_id = record.to_hash[:id]
+        job = _record_to_job(record)
+        job[Patriot::Command::PRODUCTS_ATTR] = c.select(PRODUCER_TABLE, {:job_id => serial_id}).map{|r| r.product}
+        job[Patriot::Command::REQUISITES_ATTR] = c.select(CONSUMER_TABLE, {:job_id => serial_id}).map{|r| r.product}
+        return job
+      end
+      private :_get_job
+
       # @see Patriot::JobStore::Base#get_producers
       def get_producers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
-        return _get_jobs_for_products(PRODUCER_TABLE, products, opts)
+        connect(@db_config) {|c| return _get_jobs_for_products(PRODUCER_TABLE, products, opts, c)}
       end
-  
+
       # @see Patriot::JobStore::Base#get_producers
       def get_consumers(products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
-        return _get_jobs_for_products(CONSUMER_TABLE, products, opts)
+        connect(@db_config) {|c| return _get_jobs_for_products(CONSUMER_TABLE, products, opts, c)}
       end
 
-      def _get_jobs_for_products(table, products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]})
-        result = {}
-        return result if products.nil?
+      def _get_jobs_for_products(table, products, opts = {:include_attrs => [Patriot::Command::STATE_ATTR]}, c)
+        result = []
+        return result if products.blank?
         products = [products] unless products.is_a? Array
         included_cols = (opts[:include_attrs] || []).map{|a| ATTR_TO_COLUMN[a]}
-        connect(@db_config) do |c|
-          products.each do |product|
-            jids = c.select(table, {:product => product}).map{|r| r.job_id}
-            next if jids.empty?
-            included_cols = (['job_id'] | (included_cols || [])).uniq
-            query = "SELECT job_id, #{included_cols.join(', ')} FROM jobs WHERE #{jids.map{|jid| "id = #{jid}" }.join(' OR ')}"
-            c.execute_statement(query, :select).each do |r|
-              hashval = r.to_hash
-              result[hashval.delete(:job_id)] = hashval
-            end
+
+        products.each do |product|
+          jids = c.select(table, {:product => product}).map{|r| r.job_id}
+          next if jids.empty?
+          included_cols = (['job_id'] | (included_cols || [])).uniq
+          query = "SELECT #{included_cols.join(', ')} FROM jobs WHERE #{jids.map{|jid| "id = #{jid}" }.join(' OR ')}"
+          c.execute_statement(query, :select).each do |r|
+            result.push(r.to_hash)
           end
         end
-        return result
+        return result.uniq
       end
       private :_get_jobs_for_products
 
-      # @see Patriot::JobStore::JobState
+      # @param  [String]  job_id  JOB ID
+      # @param  [Hash]    opts    options
+      # @return [Hash]    histories
       def get_execution_history(job_id, opts = {})
+        connect(@db_config) {|c| return _get_execution_history(job_id, opts, c)}
+      end
+
+      # @param  [String]  job_id  JOB ID
+      # @param  [Hash]    opts    options
+      # @param  [Patriot::Util::DBClient::Base] c
+      # @return [Hash]    histories
+      def _get_execution_history(job_id, opts, c)
         opts = {:limit => 1, :order => :DESC}.merge(opts)
         query = "SELECT * FROM #{HISTORY_TABLE} WHERE job_id = '#{job_id}' ORDER BY id #{opts[:order]} LIMIT #{opts[:limit]}"
-        connect(@db_config) do |c|
-          return c.execute_statement(query, :select).map(&:to_hash)
+
+        return c.execute_statement(query, :select).map(&:to_hash)
+      end
+      private :_get_execution_history
+
+      # get nodes and edges information to render graph
+      # @param  [String]  job_id  JOB ID
+      # @param  [Hash]    opts    options
+      # @return [Array]   [nodes, edges]
+      def get_graph(job_id, opts = {})
+        connect(@db_config) do |db_conn|
+          job = _get_job(job_id, db_conn)
+
+          job[:consumers] = _get_jobs_for_products(CONSUMER_TABLE, job[Patriot::Command::PRODUCTS_ATTR], opts, db_conn) || []
+          job[:producers] = _get_jobs_for_products(PRODUCER_TABLE, job[Patriot::Command::REQUISITES_ATTR], opts, db_conn) || []
+          history = _get_execution_history(job_id, {}, db_conn)[0]
+
+          hashed_job = {
+            :job_id => job.job_id,
+            :history => history,
+            :depth => 0
+          }.merge(job.attributes)
+
+          # set self node
+          nodes = {job_id => hashed_job}
+          edges = []
+
+          _set_dependency(
+            db_conn,
+            :producers,
+            opts[:producer_depth],
+            nodes,
+            edges,
+            hashed_job
+          )
+
+          _set_dependency(
+            db_conn,
+            :consumers,
+            opts[:consumer_depth],
+            nodes,
+            edges,
+            hashed_job
+          )
+
+          return {:nodes => nodes, :edges => edges}
         end
       end
 
+      # get dependency and set nodes and edges
+      #
+      # @private
+      # @param  [Patriot::Util::DBClient::Base] db_conn
+      # @param  [Symbol]  direction       :producers or :consumers
+      # @param  [Integer] depth           dependency depth to get
+      # @param  [Hash]    nodes           nodes to set for dager-d3
+      # @param  [Array]   edges           edges to set for dager-d3
+      # @param  [Hash]    base_job        base job to get dependency
+      def _set_dependency(db_conn, direction, depth, nodes, edges, base_job)
+        return if nodes[base_job[:job_id]][:depth] == depth
+
+        base_job[direction].map{|depend_job|
+          job = _get_job(depend_job[:job_id], db_conn)
+          job[:consumers] = _get_jobs_for_products(CONSUMER_TABLE, job[Patriot::Command::PRODUCTS_ATTR], {}, db_conn) || []
+          job[:producers] = _get_jobs_for_products(PRODUCER_TABLE, job[Patriot::Command::REQUISITES_ATTR], {}, db_conn) || []
+          history = _get_execution_history(depend_job[:job_id], {}, db_conn)[0]
+
+          hashed_job = {
+            :job_id => job.job_id,
+            :history => history,
+            :depth => base_job[:depth] + 1
+          }.merge(job.attributes)
+
+          nodes[job.job_id] = hashed_job
+          if direction == :producers
+            edges.push([job.job_id, base_job[:job_id]])
+          else
+            edges.push([base_job[:job_id], job.job_id])
+          end
+
+          # call recursively
+          _set_dependency(
+            db_conn,
+            direction,
+            depth,
+            nodes,
+            edges,
+            hashed_job
+          )
+        }
+      end
+      private :_set_dependency
+
       # @see Patriot::JobStore::Base#find_jobs_by_state
       def find_jobs_by_state(state, opts = {})
         raise "OFFSET is set WITHOUT LIMIT" if opts.has_key?(:offset) && !opts.has_key?(:limit)
@@ -339,17 +446,17 @@ END_OB_QUERY
         end
       end
 
-      def record_to_job(record)
+      def _record_to_job(record)
         job = Patriot::JobStore::Job.new(record.job_id)
         job.update_id = record.update_id
         ATTR_TO_COLUMN.each{|attr, col| job[attr] = record.send(col) }
         unless record.content.nil?
-          content = JSON.parse(record.content)
+          content = JSON.parse(record.content, {:symbolize_names => true})
           content.each{|k,v| job[k] = v}
         end
         return job
       end
-      private :record_to_job
+      private :_record_to_job
     end
   end
 end

data/lib/patriot/tool/patriot_commands/execute.rb

@@ -1,6 +1,6 @@
 module Patriot
   module Tool
-    # namesapce for patriot comman line tools
+    # namesapce for patriot command line tools
     module PatriotCommands
       # execute PBC directory
       module Execute

data/lib/patriot/tool/patriot_commands/job.rb

@@ -17,7 +17,7 @@ module Patriot
             when "delete"
               job_id.each do |jid|
                 if job_store.delete_job(jid)
-                  puts "#{jid} is deleted" 
+                  puts "#{jid} is deleted"
                 else
                   puts "#{jid} does not exist"
                 end
@@ -41,10 +41,12 @@ module Patriot
               job[Patriot::Command::REQUISITES_ATTR].each do |product|
                 products = job_store.get_producers(product)
                 values << "#{'  '*indent}<= #{product} = WARN: no producer exists" if products.empty?
-                products.each do |jid, attrs|
-                  dep_status = "#{jid}, #{attrs[Patriot::Command::STATE_ATTR]}"
+                products.each do |p|
+                  jid = p[:job_id]
+                  state = p[Patriot::Command::STATE_ATTR]
+                  dep_status = "#{jid}, #{state}"
                   producer_job = job_store.get(jid, :include_dependency => true)
-                  unless producer_job['consumers'].keys.include?(job_id)
+                  unless producer_job[:consumers].map{|c| c[:job_id]}.include?(job_id)
                     dep_status = "WARN: currupted dependency #{dep_status}"
                   end
                   values << "#{'  '*indent}<= #{product} = #{dep_status}"

data/lib/patriot/tool/patriot_commands/register.rb

@@ -27,11 +27,11 @@ module Patriot
               :type     => :boolean,
               :default  => false,
               :desc     => "don't change current state of jobs (only change definition)"
-          method_option :retry_dep, 
-              :type     => :boolean, 
+          method_option :retry_dep,
+              :type     => :boolean,
               :desc     =>  'set states of dependent jobs to WAIT'
           method_option :update_id,
-              :type     => :string,
+              :type     => :numeric,
               :default  => Time.now.to_i,
               :desc     => 'default value is current unixtime (default value is Time.now.to_i)'
           def register(date, *paths)
@@ -45,8 +45,10 @@ module Patriot
                              :store_id       => Patriot::JobStore::ROOT_STORE_ID,
                              :retry_interval => 300,
                              :retry_limite   => 10}.merge(opts)
-              opts[:state] = nil if opts[:keep_state]
-
+              if opts[:keep_state]
+                opts[:state] = nil
+                opts[:update_id] = nil
+              end
               job_store = Patriot::JobStore::Factory.create_jobstore(opts[:store_id], config)
               parser  = Patriot::Tool::BatchParser.new(config)
               jobs    = []