ood_core 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9f4136abfa563b5a8e552a4ea23fb28886d0504e
-  data.tar.gz: 8fd45406db0789e635ae41ca52021d7317eac883
+SHA256:
+  metadata.gz: 8a526602e6c6b59b6d943d299dc4e442cfd354a768669b4bc03a9423e12cf418
+  data.tar.gz: 5220c4b20c1de287afdcad2eece623952c58aea735c1786f9956912563277e85
 SHA512:
-  metadata.gz: 1cc80ddf1f1d99e10c96ade0fcc70cd6cb96638fc204dbc6ecf8533bf7753d9c80077486f1a7cb15e2270e11aba0366fdec8dbb20cfcb00e0d6bfd33f7e03894
-  data.tar.gz: e1f944deee57a03dd800bd4217b1c7f125e087edd284a87ab3dae4b655231c8b511d83e256a7df24afa66c76b72e95fd8fe60a549dd45f525ca6016fe8d21fb3
+  metadata.gz: f63f8aff330f033ef8fe0dad0d07629e3704463441f1a910920f37a86d48a4cab059182403b9cb6f1bd6a300213b1cff45315b43354fa0d2a9aaaba2f7bc54c8
+  data.tar.gz: d77d8d5130a3f20ac9e54667b10de5a476322c38491f63abe58e0ab192d23b22fc764481254abaf9def825a4ee6707646f1b3c264ed9a5ec4fa76add0f34295a

data/.gitignore

@@ -48,3 +48,6 @@ Gemfile.lock
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+# SSHFS temp files
+._*

data/CHANGELOG.md

@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.8.0] - 2019-01-29
+### Added
+- info_all_each and info_where_owner_each super class methods
+- job array support for Torque, Slurm, and SGE (currently missing from LSF and PBSPro)
+- `OodCore::Job::Status#precedence` for the ability to get an overall status for a group of jobs
+
+### Fixed
+- Fix SGE adapter to specify `-u '*'` when calling qstat to get all jobs
+
 ## [0.7.1] - 2019-01-11
 ### Fixed
 - Fixed crash when libdrmaa is used to query for a job no longer in the queue
@@ -156,7 +165,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.7.1...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/OSC/ood_core/compare/v0.7.1...v0.8.0
 [0.7.1]: https://github.com/OSC/ood_core/compare/v0.7.0...v0.7.1
 [0.7.0]: https://github.com/OSC/ood_core/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/OSC/ood_core/compare/v0.5.1...v0.6.0

data/lib/ood_core.rb

@@ -13,6 +13,7 @@ module OodCore
     require "ood_core/job/status"
     require "ood_core/job/adapter"
     require "ood_core/job/factory"
+    require "ood_core/job/task"
 
     # A namespace for job adapters
     # @note These are dynamically loaded upon request

data/lib/ood_core/job/adapter.rb

@@ -36,18 +36,80 @@ module OodCore
       # Retrieve info for all jobs from the resource manager
       # @abstract Subclass is expected to implement {#info_all}
       # @raise [NotImplementedError] if subclass did not define {#info_all}
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      #   This array specifies only attrs you want, in addition to id and status.
+      #   If an array, the Info object that is returned to you is not guarenteed
+      #   to have a value for any attr besides the ones specified and id and status.
+      #
+      #   For certain adapters this may speed up the response since
+      #   adapters can get by without populating the entire Info object
       # @return [Array<Info>] information describing submitted jobs
-      def info_all
+      def info_all(attrs: nil)
         raise NotImplementedError, "subclass did not define #info_all"
       end
 
       # Retrieve info for all jobs for a given owner or owners from the
       # resource manager
       # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      #   This array specifies only attrs you want, in addition to id and status.
+      #   If an array, the Info object that is returned to you is not guarenteed
+      #   to have a value for any attr besides the ones specified and id and status.
+      #
+      #   For certain adapters this may speed up the response since
+      #   adapters can get by without populating the entire Info object
       # @return [Array<Info>] information describing submitted jobs
-      def info_where_owner(owner)
+      def info_where_owner(owner, attrs: nil)
         owner = Array.wrap(owner).map(&:to_s)
-        info_all.select { |info| owner.include? info.job_owner }
+
+        # must at least have job_owner to filter by job_owner
+        attrs = Array.wrap(attrs) | [:job_owner] unless attrs.nil?
+
+        info_all(attrs: attrs).select { |info| owner.include? info.job_owner }
+      end
+
+      # Iterate over each job Info object
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      #   This array specifies only attrs you want, in addition to id and status.
+      #   If an array, the Info object that is returned to you is not guarenteed
+      #   to have a value for any attr besides the ones specified and id and status.
+      #
+      #   For certain adapters this may speed up the response since
+      #   adapters can get by without populating the entire Info object
+      # @yield [Info] of each job to block
+      # @return [Enumerator] if no block given
+      def info_all_each(attrs: nil)
+        return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+        info_all(attrs: attrs).each do |job|
+          yield job
+        end
+      end
+
+      # Iterate over each job Info object
+      # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      #   This array specifies only attrs you want, in addition to id and status.
+      #   If an array, the Info object that is returned to you is not guarenteed
+      #   to have a value for any attr besides the ones specified and id and status.
+      #
+      #   For certain adapters this may speed up the response since
+      #   adapters can get by without populating the entire Info object
+      # @yield [Info] of each job to block
+      # @return [Enumerator] if no block given
+      def info_where_owner_each(owner, attrs: nil)
+        return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+        info_where_owner(owner, attrs: attrs).each do |job|
+          yield job
+        end
+      end
+
+      # Whether the adapter supports job arrays
+      # @return [Boolean] - assumes true; but can be overridden by adapters that
+      #   explicitly do not
+      def supports_job_arrays?
+        true
       end
 
       # Retrieve job info from the resource manager

data/lib/ood_core/job/adapters/lsf.rb

@@ -108,7 +108,7 @@ module OodCore
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
         # @see Adapter#info_all
-        def info_all
+        def info_all(attrs: nil)
           batch.get_jobs.map { |v| info_for_batch_hash(v) }
         rescue Batch::Error => e
           raise JobAdapterError, e.message
@@ -118,7 +118,7 @@ module OodCore
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
         # @see Adapter#info_where_owner
-        def info_where_owner(owner)
+        def info_where_owner(owner, attrs: nil)
           owners = Array.wrap(owner).map(&:to_s)
           if owners.count > 1
             super
@@ -131,6 +131,10 @@ module OodCore
           raise JobAdapterError, e.message
         end
 
+        def supports_job_arrays?
+          false
+        end
+
         # Retrieve job status from resource manager
         # @param id [#to_s] the id of the job
         # @raise [JobAdapterError] if something goes wrong getting job status

data/lib/ood_core/job/adapters/pbspro.rb

@@ -279,7 +279,7 @@ module OodCore
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
         # @see Adapter#info_all
-        def info_all
+        def info_all(attrs: nil)
           @pbspro.get_jobs.map do |v|
             parse_job_info(v)
           end
@@ -292,7 +292,7 @@ module OodCore
       # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
       # @raise [JobAdapterError] if something goes wrong getting job info
       # @return [Array<Info>] information describing submitted jobs
-      def info_where_owner(owner)
+      def info_where_owner(owner, attrs: nil)
         owner = Array.wrap(owner).map(&:to_s)
 
         usr_jobs = @pbspro.select_jobs(args: ["-u", owner.join(",")])
@@ -307,6 +307,10 @@ module OodCore
         end
       end
 
+        def supports_job_arrays?
+          false
+        end
+
         # Retrieve job info from the resource manager
         # @param id [#to_s] the id of the job
         # @raise [JobAdapterError] if something goes wrong getting job info

data/lib/ood_core/job/adapters/sge.rb

@@ -90,8 +90,8 @@ module OodCore
 
         # Retrieve info for all jobs from the resource manager
         # @return [Array<Info>] information describing submitted jobs
-        def info_all
-          @batch.get_all
+        def info_all(attrs: nil)
+          @batch.get_all(owner: '*')
         rescue Batch::Error => e
           raise JobAdapterError, e.message
         end
@@ -101,7 +101,7 @@ module OodCore
         # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
-        def info_where_owner(owner)
+        def info_where_owner(owner, attrs: nil)
           owner = Array.wrap(owner).map(&:to_s).join(',')
           @batch.get_all(owner: owner)
         rescue Batch::Error => e
@@ -160,4 +160,4 @@ module OodCore
       end
     end
   end
-end
+end

data/lib/ood_core/job/adapters/sge/batch.rb

@@ -95,13 +95,7 @@ class OodCore::Job::Adapters::Sge::Batch
 
       job_hash = listener.parsed_job
 
-      if can_use_drmaa?
-        begin
-          job_hash[:status] = get_status_from_drmma(job_id)
-        rescue DRMAA::DRMAAInvalidArgumentError => e
-          raise Error, e.message
-        end
-      end
+      update_job_hash_status!(job_hash)
 
       job_info = OodCore::Job::Info.new(**job_hash)
     rescue REXML::ParseException => e
@@ -117,6 +111,22 @@ class OodCore::Job::Adapters::Sge::Batch
     job_info
   end
 
+  def update_job_hash_status!(job_hash)
+    if get_status_from_drmaa?(job_hash)
+      begin
+        job_hash[:status] = get_status_from_drmma(job_hash[:id])
+      rescue DRMAA::DRMAAInvalidArgumentError => e
+        raise Error, e.message
+      end
+    end
+  end
+
+  def get_status_from_drmaa?(job_hash)
+    # DRMAA does not recognize the parent task in job arrays
+    # e.g. 123 is invalid if it is an array job, while 123.4 is valid
+    can_use_drmaa? && job_hash[:tasks].empty?
+  end
+
   def can_use_drmaa?
     @can_use_drmaa
   end
@@ -212,6 +222,7 @@ class OodCore::Job::Adapters::Sge::Batch
     end
 
     job_hash[:status] = translate_sge_state(job_hash[:status])
+    update_job_hash_status!(job_hash)
 
     job_hash
   end

data/lib/ood_core/job/adapters/sge/helper.rb

@@ -41,6 +41,7 @@ class OodCore::Job::Adapters::Sge::Helper
     args += ['-a', script.start_time.strftime('%C%y%m%d%H%M.%S')] unless script.start_time.nil?
     args += ['-l', "h_rt=" + seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
     args += ['-P', script.accounting_id] unless script.accounting_id.nil?
+    args += ['-t', script.job_array_request] unless script.job_array_request.nil?
     args += Array.wrap(script.native) if script.native
 
     args

data/lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb

@@ -1,6 +1,8 @@
 require 'rexml/document'
 require 'rexml/streamlistener'
 require 'date'
+require 'ood_core'
+require 'ood_core/job/array_ids'
 
 # An XML stream listener to build an array of OodCore::Job::Info from qstat output
 #
@@ -13,9 +15,7 @@ require 'date'
 # :queue_name
 # :status
 # :wallclock_limit
-
-
-# :wallclock_time  # HOW LONG HAS IT BEEN RUNNING?
+# :wallclock_time
 
 class QstatXmlJRListener
   # [Hash]
@@ -25,12 +25,28 @@ class QstatXmlJRListener
 
   def initialize
     @parsed_job = {
+      :tasks => [],
       :status => :queued,
       :procs => 1,  # un-knowable from SGE qstat output
       :native => {}  # TODO: improve native attribute reporting
     }
     @current_text = nil
     @current_request = nil
+
+    @processing_job_array_spec = false
+    @job_array_spec = {
+      start: nil,
+      stop: nil,
+      step: 1,  # Step can have a default of 1
+    }
+    @running_tasks = []
+  end
+
+  def tag_start(name, attrs)
+    case name
+    when 'task_id_range'
+      toggle_processing_array_spec
+    end
   end
 
   def tag_end(name)
@@ -57,6 +73,18 @@ class QstatXmlJRListener
       end_CE_stringval
     when 'QR_name'
       end_QR_name
+    when 'JAT_task_number'
+      end_JAT_task_number
+    when 'djob_info'
+      finalize_parsed_job
+    when 'RN_min'
+      set_job_array_piece(:start)
+    when 'RN_max'
+      set_job_array_piece(:stop)
+    when 'RN_step'
+      set_job_array_piece(:step)
+    when 'task_id_range'
+      toggle_processing_array_spec
     end
   end
 
@@ -112,5 +140,51 @@ class QstatXmlJRListener
   def end_QR_name
     @parsed_job[:queue_name] = @current_text
   end
+
+  # Used to record a running Job Array task
+  def end_JAT_task_number
+    @running_tasks << @current_text
+  end
+
+  def set_job_array_piece(key)
+    @job_array_spec[key] = @current_text if @processing_job_array_spec
+  end
+
+  def spec_string
+    # If any of the job_array_spec values are nil then return a default spec_string
+    if @job_array_spec.values.any? { |value| value.nil? } 
+      '1-1:1'
+    else
+      '%{start}-%{stop}:%{step}' % @job_array_spec
+    end
+  end
+
+  def build_tasks
+    all_task_ids = OodCore::Job::ArrayIds.new(spec_string).ids
+    highest_id_running = @running_tasks.sort.last.to_i
+    
+    @running_tasks.sort.map{
+      |task_id| { :id => task_id, :status => :running }
+    } + all_task_ids.select{
+      |task_id| task_id > highest_id_running
+    }.map{
+      |task_id| { :id => task_id, :status => :queued }
+    }
+  end
+
+  # Used to finalize the parsed job
+  def finalize_parsed_job
+    @parsed_job[:tasks] = build_tasks if need_to_build_job_array?
+  end
+
+  # The XML output will always contain nodes for task_id_range, even when the
+  # job is not an array job.
+  def need_to_build_job_array?
+    spec_string != '1-1:1'
+  end
+
+  def toggle_processing_array_spec
+    @processing_job_array_spec = ! @processing_job_array_spec
+  end
 end
 

data/lib/ood_core/job/adapters/sge/qstat_xml_r_listener.rb

@@ -23,6 +23,7 @@ class QstatXmlRListener
   def initialize
     @parsed_jobs = []
     @current_job = {
+      :tasks => [],
       :native => {}  # TODO: improve native reporting
     }
     @current_text = nil
@@ -61,6 +62,8 @@ class QstatXmlRListener
       end_JAT_start_time
     when 'hard_request'
       end_hard_request
+    when 'tasks'
+      add_child_tasks
     end
   end
 
@@ -131,8 +134,15 @@ class QstatXmlRListener
   def end_job_list
     @parsed_jobs << @current_job
     @current_job = {
+      :tasks => [],
       :native => {}
     }
   end
+
+  def add_child_tasks
+    @current_job[:tasks] = OodCore::Job::ArrayIds.new(@current_text).ids.sort.map{
+      |task_id| { :id => task_id, :status => :queued }
+    }
+  end
 end
 

data/lib/ood_core/job/adapters/slurm.rb

@@ -292,6 +292,7 @@ module OodCore
           args += ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
           args += ["-A", script.accounting_id] unless script.accounting_id.nil?
           args += ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
+          args += ['-a', script.job_array_request] unless script.job_array_request.nil?
           # ignore nodes, don't know how to do this for slurm
 
           # Set dependencies
@@ -326,7 +327,7 @@ module OodCore
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
         # @see Adapter#info_all
-        def info_all
+        def info_all(attrs: nil)
           @slurm.get_jobs.map do |v|
             parse_job_info(v)
           end
@@ -345,13 +346,8 @@ module OodCore
             parse_job_info(v)
           end
 
-          # A job id can return multiple jobs if it corresponds to a job
-          # array id, so we need to find the job that corresponds to the
-          # given job id (if we can't find it, we assume it has completed)
-          info_ary.detect( -> { Info.new(id: id, status: :completed) } ) do |info|
-            # Match the job id or the formatted job & task id "1234_0"
-            info.id == id || info.native[:array_job_task_id] == id
-          end
+          # If no job was found we assume that it has completed
+          info_ary.empty? ? Info.new(id: id, status: :completed) : handle_job_array(info_ary, id)
         rescue Batch::Error => e
           # set completed status if can't find job id
           if /Invalid job id specified/ =~ e.message
@@ -500,6 +496,24 @@ module OodCore
               native: v
             )
           end
+
+          def handle_job_array(info_ary, id)
+            # If only one job was returned we return it
+            return info_ary.first unless info_ary.length > 1
+            
+            parent_task_hash = {:tasks => []}
+
+            info_ary.map do |task_info|
+              parent_task_hash[:tasks] << {:id => task_info.id, :status => task_info.status}
+
+              if task_info.id == id || task_info.native[:array_job_task_id] == id
+                # Merge hashes without clobbering the child tasks
+                parent_task_hash.merge!(task_info.to_h.select{|k, v| k != :tasks})
+              end
+            end
+
+            Info.new(**parent_task_hash)
+          end
       end
     end
   end

data/lib/ood_core/job/adapters/torque.rb

@@ -113,6 +113,7 @@ module OodCore
             headers.merge!(Execution_Time: script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")) unless script.start_time.nil?
             headers.merge!(Account_Name: script.accounting_id) unless script.accounting_id.nil?
             headers.merge!(depend: depend.join(','))       unless depend.empty?
+            headers.merge!(job_array_request: script.job_array_request) unless script.job_array_request.nil?
 
             # Set resources
             resources = {}
@@ -150,7 +151,7 @@ module OodCore
             args += ["-A", script.accounting_id] unless script.accounting_id.nil?
             args += ["-W", "depend=#{depend.join(",")}"] unless depend.empty?
             args += ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
-
+            args += ['-t', script.job_array_request] unless script.job_array_request.nil?
             # Set environment variables
             env = script.job_environment.to_h
             args += ["-v", env.keys.join(",")] unless env.empty?
@@ -173,7 +174,7 @@ module OodCore
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
         # @see Adapter#info_all
-        def info_all
+        def info_all(attrs: nil)
           @pbs.get_jobs.map do |k, v|
             parse_job_info(k, v)
           end
@@ -186,7 +187,7 @@ module OodCore
         # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
-        def info_where_owner(owner)
+        def info_where_owner(owner, attrs: nil)
           owner = Array.wrap(owner).map(&:to_s)
           @pbs.select_jobs(
             attribs: [
@@ -206,7 +207,13 @@ module OodCore
         # @see Adapter#info
         def info(id)
           id = id.to_s
-          parse_job_info(*@pbs.get_job(id).flatten)
+          result = @pbs.get_job(id)
+
+          if result.keys.length == 1
+            parse_job_info(*result.flatten)
+          else
+            parse_job_array(id, result)
+          end
         rescue Torque::FFI::UnkjobidError
           # set completed status if can't find job id
           Info.new(
@@ -224,8 +231,13 @@ module OodCore
         # @see Adapter#status
         def status(id)
           id = id.to_s
-          char = @pbs.get_job(id, filters: [:job_state])[id][:job_state]
-          Status.new(state: STATE_MAP.fetch(char, :undetermined))
+          @pbs.get_job(id, filters: [:job_state]).values.map {
+            |job_status| OodCore::Job::Status.new(
+              state: STATE_MAP.fetch(
+                job_status[:job_state], :undetermined
+              )
+            )
+          }.max
         rescue Torque::FFI::UnkjobidError
           # set completed status if can't find job id
           Status.new(state: :completed)
@@ -300,8 +312,31 @@ module OodCore
             end
           end
 
+          def parse_job_array(parent_id, result)
+            results = result.to_a
+
+            parse_job_info(
+              parent_id,
+              results.first.last.tap { |info_hash| info_hash[:exec_host] = aggregate_exec_host(results) },
+              tasks: generate_task_list(results)
+            )
+          end
+
+          def aggregate_exec_host(results)
+            results.map { |k,v| v[:exec_host] }.compact.sort.uniq.join("+")
+          end
+
+          def generate_task_list(results)
+            results.map do |k, v|
+              {
+                :id => k,
+                :status => STATE_MAP.fetch(v[:job_state], :undetermined)
+              }
+            end
+          end
+
           # Parse hash describing PBS job status
-          def parse_job_info(k, v)
+          def parse_job_info(k, v, tasks: [])
             /^(?<job_owner>[\w-]+)@/ =~ v[:Job_Owner]
             allocated_nodes = parse_nodes(v[:exec_host] || "")
             procs = allocated_nodes.inject(0) { |sum, x| sum + x[:procs] }
@@ -329,7 +364,8 @@ module OodCore
               cpu_time: duration_in_seconds(v.fetch(:resources_used, {})[:cput]),
               submission_time: v[:ctime],
               dispatch_time: v[:start_time],
-              native: v
+              native: v,
+              tasks: tasks
             )
           end
       end

data/lib/ood_core/job/array_ids.rb

@@ -0,0 +1,55 @@
+# Builds a sorted array of job ids given a job array spec string
+# 
+# Job array spec strings:
+#   1         Single id
+#   1-10      Range
+#   1-10:2    Range with step
+#   1-10,13   Compound (range with single id)
+#   
+# Note that Ranges are expected to be inclusive
+module OodCore
+  module Job
+    class ArrayIds
+      attr_reader :ids
+      def initialize(spec_string)
+        @ids = []
+        parse_spec_string(spec_string)
+      end
+
+      protected
+      def parse_spec_string(spec_string)
+        @ids = get_components(spec_string).map{
+          |component| process_component(component)
+        }.reduce(:+).sort
+      end
+
+      def get_components(spec_string)
+        discard_percent_modifier(spec_string).split(',')
+      end
+
+      # A few adapters use percent to define an arrays maximum number of
+      # simultaneous tasks. The percent is expected to come at the end.
+      def discard_percent_modifier(spec_string)
+        spec_string.split('%').first
+      end
+
+      def process_component(component)
+        is_range?(component) ? get_range(component) : [ component.to_i ]
+      end
+
+      def get_range(component)
+        raw_range, raw_step = component.split(':')
+        start, stop = raw_range.split('-').map(&:to_i)
+        range = Range.new(start, stop)
+        step = raw_step.to_i
+        step = 1 if step == 0
+        
+        range.step(step).to_a
+      end
+
+      def is_range?(component)
+        component.include?('-')
+      end
+    end
+  end
+end

data/lib/ood_core/job/info.rb

@@ -65,6 +65,11 @@ module OodCore
       # @return [Object] native info
       attr_reader :native
 
+      # List of job array child task statuses
+      # @note only relevant for job arrays
+      # @return [Array<Task>] tasks
+      attr_reader :tasks
+
       # @param id [#to_s] job id
       # @param status [#to_sym] job state
       # @param allocated_nodes [Array<#to_h>] allocated nodes
@@ -79,12 +84,14 @@ module OodCore
       # @param cpu_time [#to_i, nil] cpu time
       # @param submission_time [#to_i, nil] submission time
       # @param dispatch_time [#to_i, nil] dispatch time
+      # @param tasks [Array<Hash>] tasks e.g. { id: '12345.owens-batch', status: :running }
       # @param native [Object] native info
       def initialize(id:, status:, allocated_nodes: [], submit_host: nil,
                      job_name: nil, job_owner: nil, accounting_id: nil,
                      procs: nil, queue_name: nil, wallclock_time: nil,
                      wallclock_limit: nil, cpu_time: nil, submission_time: nil,
-                     dispatch_time: nil, native: nil, **_)
+                     dispatch_time: nil, native: nil, tasks: [],
+                     **_)
         @id              = id.to_s
         @status          = Status.new(state: status.to_sym)
         @allocated_nodes = allocated_nodes.map { |n| NodeInfo.new(n.to_h) }
@@ -99,6 +106,10 @@ module OodCore
         @cpu_time        = cpu_time        && cpu_time.to_i
         @submission_time = submission_time && Time.at(submission_time.to_i)
         @dispatch_time   = dispatch_time   && Time.at(dispatch_time.to_i)
+        @tasks = tasks.map {|task_status| Task.new(**task_status)}
+
+        @status = job_array_aggregate_status unless @tasks.empty?
+
         @native          = native
       end
 
@@ -120,7 +131,8 @@ module OodCore
           cpu_time:        cpu_time,
           submission_time: submission_time,
           dispatch_time:   dispatch_time,
-          native:          native
+          native:          native,
+          tasks: tasks
         }
       end
 
@@ -143,6 +155,14 @@ module OodCore
       def hash
         [self.class, to_h].hash
       end
+
+      private
+
+      # Generate an aggregate status from child tasks
+      # @return [OodCore::Job::Status]
+      def job_array_aggregate_status
+        @tasks.map { |task_status| task_status.status }.max
+      end
     end
   end
 end

data/lib/ood_core/job/script.rb

@@ -95,6 +95,10 @@ module OodCore
       # @return [String, nil] accounting id
       attr_reader :accounting_id
 
+      # The job array request, commonly in the format '$START-$STOP'
+      # @return [String, nil] job array request
+      attr_reader :job_array_request
+
       # Object detailing any native specifications that are implementation specific
       # @note Should not be used at all costs.
       # @return [Object, nil] native specifications
@@ -128,7 +132,7 @@ module OodCore
                      job_name: nil, shell_path: nil, input_path: nil,
                      output_path: nil, error_path: nil, reservation_id: nil,
                      queue_name: nil, priority: nil, start_time: nil,
-                     wall_time: nil, accounting_id: nil, native: nil, **_)
+                     wall_time: nil, accounting_id: nil, job_array_request: nil, native: nil, **_)
         @content = content.to_s
 
         @submit_as_hold      = submit_as_hold
@@ -136,22 +140,23 @@ module OodCore
         @email_on_started    = email_on_started
         @email_on_terminated = email_on_terminated
 
-        @args             = args            && args.map(&:to_s)
-        @job_environment  = job_environment && job_environment.each_with_object({}) { |(k, v), h| h[k.to_s] = v.to_s }
-        @workdir          = workdir         && Pathname.new(workdir.to_s)
-        @email            = email           && Array.wrap(email).map(&:to_s)
-        @job_name         = job_name        && job_name.to_s
-        @shell_path       = shell_path      && Pathname.new(shell_path.to_s)
-        @input_path       = input_path      && Pathname.new(input_path.to_s)
-        @output_path      = output_path     && Pathname.new(output_path.to_s)
-        @error_path       = error_path      && Pathname.new(error_path.to_s)
-        @reservation_id   = reservation_id  && reservation_id.to_s
-        @queue_name       = queue_name      && queue_name.to_s
-        @priority         = priority        && priority.to_i
-        @start_time       = start_time      && Time.at(start_time.to_i)
-        @wall_time        = wall_time       && wall_time.to_i
-        @accounting_id    = accounting_id   && accounting_id.to_s
-        @native           = native
+        @args               = args              && args.map(&:to_s)
+        @job_environment    = job_environment   && job_environment.each_with_object({}) { |(k, v), h| h[k.to_s] = v.to_s }
+        @workdir            = workdir           && Pathname.new(workdir.to_s)
+        @email              = email             && Array.wrap(email).map(&:to_s)
+        @job_name           = job_name          && job_name.to_s
+        @shell_path         = shell_path        && Pathname.new(shell_path.to_s)
+        @input_path         = input_path        && Pathname.new(input_path.to_s)
+        @output_path        = output_path       && Pathname.new(output_path.to_s)
+        @error_path         = error_path        && Pathname.new(error_path.to_s)
+        @reservation_id     = reservation_id    && reservation_id.to_s
+        @queue_name         = queue_name        && queue_name.to_s
+        @priority           = priority          && priority.to_i
+        @start_time         = start_time        && Time.at(start_time.to_i)
+        @wall_time          = wall_time         && wall_time.to_i
+        @accounting_id      = accounting_id     && accounting_id.to_s
+        @job_array_request  = job_array_request && job_array_request.to_s
+        @native             = native
       end
 
       # Convert object to hash
@@ -178,6 +183,7 @@ module OodCore
           start_time:          start_time,
           wall_time:           wall_time,
           accounting_id:       accounting_id,
+          job_array_request:   job_array_request,
           native:              native
         }
       end

data/lib/ood_core/job/status.rb

@@ -22,14 +22,16 @@ module OodCore
         #
         #   # Job is completed and not running on an execution host
         #   :completed
+        #   
+        #   @note that this list's order is meaningful and should not be sorted lexigraphically
         def states
           %i(
             undetermined
-            queued
+            completed
             queued_held
+            queued
             running
             suspended
-            completed
           )
         end
       end
@@ -113,6 +115,17 @@ module OodCore
           self == state
         end
       end
+
+      def precedence
+        self.class.states.index(@state)
+      end
+
+      # The comparison operator for sorting values.
+      #
+      # @return [Integer] Comparison value based on precedence
+      def <=>(other)
+        precedence <=> other.precedence
+      end
     end
   end
 end

data/lib/ood_core/job/task.rb

@@ -0,0 +1,24 @@
+module OodCore
+  module Job
+    class Task
+      attr_reader :id
+      attr_reader :status
+
+      def initialize(id:, status:, **_)
+        @task_id = id
+        @status = OodCore::Job::Status.new(state: status)
+      end
+
+      def to_h
+        {
+          :id => id,
+          :status => status
+        }
+      end
+
+      def ==(other)
+        self.to_h == other.to_h
+      end
+    end
+  end
+end

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.7.1"
+  VERSION = "0.8.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-11 00:00:00.000000000 Z
+date: 2019-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -147,6 +147,7 @@ files:
 - lib/ood_core/cluster.rb
 - lib/ood_core/clusters.rb
 - lib/ood_core/errors.rb
+- lib/ood_core/job/._task_status.rb
 - lib/ood_core/job/adapter.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
@@ -165,11 +166,13 @@ files:
 - lib/ood_core/job/adapters/torque/batch.rb
 - lib/ood_core/job/adapters/torque/error.rb
 - lib/ood_core/job/adapters/torque/ffi.rb
+- lib/ood_core/job/array_ids.rb
 - lib/ood_core/job/factory.rb
 - lib/ood_core/job/info.rb
 - lib/ood_core/job/node_info.rb
 - lib/ood_core/job/script.rb
 - lib/ood_core/job/status.rb
+- lib/ood_core/job/task.rb
 - lib/ood_core/refinements/array_extensions.rb
 - lib/ood_core/refinements/drmaa_extensions.rb
 - lib/ood_core/refinements/hash_extensions.rb
@@ -195,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Open OnDemand core library