ood_core 0.5.1 → 0.6.0

data/lib/ood_core/job/adapters/sge.rb

@@ -0,0 +1,163 @@
+require "ood_core/refinements/array_extensions"
+require "ood_core/refinements/hash_extensions"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      # Build the Sun Grid Engine adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :cluster (nil) The cluster to communicate with
+      # @option config [Object] :conf (nil) Path to the SGE conf
+      # @option config [Object] :bin (nil) Path to SGE client binaries
+      # @option config [Object] :sge_root (nil) Path to SGE root, note that
+      # @option config [#to_h] :bin_overrides ({}) Optional overrides to SGE client executables
+      #   this may be nil, but must be set to use the DRMAA API, and there is a
+      #   severe performance penalty calling Sge#info without using DRMAA.
+      def self.build_sge(config)
+        batch = Adapters::Sge::Batch.new(config.to_h.symbolize_keys)
+        Adapters::Sge.new(batch: batch)
+      end
+    end
+
+    module Adapters
+      class Sge < Adapter
+        using Refinements::HashExtensions
+        using Refinements::ArrayExtensions
+
+        require "ood_core/job/adapters/sge/batch"
+        require "ood_core/job/adapters/sge/helper"
+
+        # The cluster of the Sun Grid Engine batch server
+        # @example UCLA's hoffman2 cluster
+        #   my_batch.cluster #=> "hoffman2"
+        # @return [String, nil] the cluster name
+        attr_reader :cluster
+
+        # The path to the Sun Grid Engine configuration file
+        # @example For Sun Grid Engine 8.0.1
+        #   my_batch.conf.to_s #=> "/u/systems/UGE8.0.1vm/h2.conf
+        # @return [Pathname, nil] path to gridengine conf
+        attr_reader :conf
+
+        # The path to the Sun Grid Engine client installation binaries
+        # @example For Sun Grid Engine 8.0.1
+        #   my_batch.bin.to_s #=> "/u/systems/UGE8.0.1vm/bin/lx-amd64/
+        # @return [Pathname] path to SGE binaries
+        attr_reader :bin
+
+        # The root exception class that all Sun Grid Engine-specific exceptions inherit
+        # from
+        class Error < StandardError; end
+
+        # @param batch [Adapters::Sge::Batch]
+        def initialize(batch:)
+          @batch = batch
+          @helper = Sge::Helper.new
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @example Submit job template to cluster
+        #   solver_id = job_adapter.submit(solver_script)
+        #   #=> "1234.server"
+        # @example Submit job that depends on previous job
+        #   post_id = job_adapter.submit(
+        #     post_script,
+        #     afterok: solver_id
+        #   )
+        #   #=> "1235.server"
+        # @param script [Script] script object that describes the
+        #   script and attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] this job may be scheduled for execution
+        #   at any point after dependent jobs have started execution
+        # @param afterok [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution only after dependent jobs have terminated with no errors
+        # @param afternotok [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution only after dependent jobs have terminated with errors
+        # @param afterany [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution after dependent jobs have terminated
+        # @raise [JobAdapterError] if something goes wrong submitting a job
+        # @return [String] the job id returned after successfully submitting a job
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          # SGE supports jod dependencies on job completion
+          args = @helper.batch_submit_args(script, after: after, afterok: afterok, afternotok: afternotok, afterany: afterany)
+
+          @batch.submit(script.content, args)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs from the resource manager
+        # @return [Array<Info>] information describing submitted jobs
+        def info_all
+          @batch.get_all
+        rescue Batch::Error => e
+            raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs for a given owner or owners from the
+        # resource manager
+        # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Array<Info>] information describing submitted jobs
+        def info_where_owner(owner)
+          owner = Array.wrap(owner).map(&:to_s).join(',')
+          @batch.get_all(owner: owner)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve job info from the resource manager
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Info] information describing submitted job
+        def info(id)
+          @batch.get_info_enqueued_job(id)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve job status from resource manager
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong getting the status of a job
+        # @return [Status] status of job
+        def status(id)
+          info(id).status
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Put the submitted job on hold
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong holding a job
+        # @return [void]
+        def hold(id)
+          @batch.hold(id.to_s)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Release the job that is on hold
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong releasing a job
+        # @return [void]
+        def release(id)
+          @batch.release(id.to_s)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Delete the submitted job
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong deleting a job
+        # @return [void]
+        def delete(id)
+          @batch.delete(id.to_s)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/job/adapters/slurm.rb

@@ -1,5 +1,6 @@
 require "time"
 require "ood_core/refinements/hash_extensions"
+require "ood_core/job/adapters/helper"
 
 module OodCore
   module Job
@@ -11,12 +12,14 @@ module OodCore
       # @option config [Object] :cluster (nil) The cluster to communicate with
       # @option config [Object] :conf (nil) Path to the slurm conf
       # @option config [Object] :bin (nil) Path to slurm client binaries
+      # @option config [#to_h] :bin_overrides ({}) Optional overrides to Slurm client executables
       def self.build_slurm(config)
         c = config.to_h.symbolize_keys
         cluster = c.fetch(:cluster, nil)
         conf    = c.fetch(:conf, nil)
         bin     = c.fetch(:bin, nil)
-        slurm = Adapters::Slurm::Batch.new(cluster: cluster, conf: conf, bin: bin)
+        bin_overrides = c.fetch(:bin_overrides, {})
+        slurm = Adapters::Slurm::Batch.new(cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides)
         Adapters::Slurm.new(slurm: slurm)
       end
     end
@@ -48,6 +51,12 @@ module OodCore
           # @return [Pathname] path to slurm binaries
           attr_reader :bin
 
+          # Optional overrides for Slurm client executables
+          # @example
+          #  {'sbatch' => '/usr/local/bin/sbatch'}
+          # @return Hash<String, String>
+          attr_reader :bin_overrides
+
           # The root exception class that all Slurm-specific exceptions inherit
           # from
           class Error < StandardError; end
@@ -55,10 +64,11 @@ module OodCore
           # @param cluster [#to_s, nil] the cluster name
           # @param conf [#to_s, nil] path to the slurm conf
           # @param bin [#to_s] path to slurm installation binaries
-          def initialize(cluster: nil, bin: nil, conf: nil)
+          def initialize(cluster: nil, bin: nil, conf: nil, bin_overrides: {})
             @cluster = cluster && cluster.to_s
             @conf    = conf    && Pathname.new(conf.to_s)
             @bin     = Pathname.new(bin.to_s)
+            @bin_overrides = bin_overrides
           end
 
           # Get a list of hashes detailing each of the jobs on the batch server
@@ -140,7 +150,7 @@ module OodCore
           private
             # Call a forked Slurm command for a given cluster
             def call(cmd, *args, env: {}, stdin: "")
-              cmd = bin.join(cmd.to_s).to_s
+              cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
               args  = args.map(&:to_s)
               args += ["-M", cluster] if cluster
               env = env.to_h
@@ -150,11 +160,11 @@ module OodCore
             end
 
             # Fields requested from a formatted `squeue` call
+            # Note that the order of these fields is important
             def fields
               {
                 account: "%a",
                 job_id: "%A",
-                gres: "%b",
                 exec_host: "%B",
                 min_cpus: "%c",
                 cpus: "%C",
@@ -200,7 +210,8 @@ module OodCore
                 nice: "%y",
                 scheduled_nodes: "%Y",
                 sockets_cores_threads: "%z",
-                work_dir: "%Z"
+                work_dir: "%Z",
+                gres: "%b",  # must come at the end to fix a bug with Slurm 18
               }
             end
         end

data/lib/ood_core/job/adapters/torque/attributes.rb

@@ -0,0 +1,109 @@
+class OodCore::Job::Adapters::Torque
+  # Maintains a constant Hash of defined PBS attribute types
+  # Includes:
+  #   Attribute names used by user commands
+  #   Additional job and general attribute names
+  #   Additional queue attribute names
+  #   Additional server attribute names
+  #   Additional node attribute names
+  ATTR = {
+    # Attribute names used by user commands
+    a:                 :Execution_Time,
+    c:                 :Checkpoint,
+    e:                 :Error_Path,
+    f:                 :fault_tolerant,
+    g:                 :group_list,
+    h:                 :Hold_Types,
+    j:                 :Join_Path,
+    k:                 :Keep_Files,
+    l:                 :Resource_List,
+    m:                 :Mail_Points,
+    o:                 :Output_Path,
+    p:                 :Priority,
+    q:                 :destination,
+    r:                 :Rerunable,
+    t:                 :job_array_request,
+    array_id:          :job_array_id,
+    u:                 :User_List,
+    v:                 :Variable_List,
+    A:                 :Account_Name,
+    args:              :job_arguments,
+    reservation_id:    :reservation_id,
+    login_node_id:     :login_node_id,
+    login_prop:        :login_property,
+    external_nodes:    :external_nodes,
+    multi_req_alps:    :multi_req_alps,
+    M:                 :Mail_Users,
+    N:                 :Job_Name,
+    S:                 :Shell_Path_List,
+    depend:            :depend,
+    inter:             :interactive,
+    stagein:           :stagein,
+    stageout:          :stageout,
+    jobtype:           :jobtype,
+    submit_host:       :submit_host,
+    init_work_dir:     :init_work_dir,
+
+    # Additional job and general attribute names
+    ctime:             :ctime,
+    exechost:          :exec_host,
+    execport:          :exec_port,
+    mtime:             :mtime,
+    qtime:             :qtime,
+    session:           :session_id,
+    euser:             :euser,
+    egroup:            :egroup,
+    hashname:          :hashname,
+    hopcount:          :hop_count,
+    security:          :security,
+    sched_hint:        :sched_hint,
+    substate:          :substate,
+    name:              :Job_Name,
+    owner:             :Job_Owner,
+    used:              :resources_used,
+    state:             :job_state,
+    queue:             :queue,
+    server:            :server,
+    maxrun:            :max_running,
+    maxreport:         :max_report,
+    total:             :total_jobs,
+    comment:           :comment,
+    cookie:            :cookie,
+    qrank:             :queue_rank,
+    altid:             :alt_id,
+    etime:             :etime,
+    exitstat:          :exit_status,
+    forwardx11:        :forward_x11,
+    submit_args:       :submit_args,
+    tokens:            :tokens,
+    netcounter:        :net_counter,
+    umask:             :umask,
+    start_time:        :start_time,
+    start_count:       :start_count,
+    checkpoint_dir:    :checkpoint_dir,
+    checkpoint_name:   :checkpoint_name,
+    checkpoint_time:   :checkpoint_time,
+    checkpoint_restart_status:  :checkpoint_restart_status,
+    restart_name:      :restart_name,
+    comp_time:         :comp_time,
+    reported:          :reported,
+    intcmd:            :inter_cmd,
+    job_radix:         :job_radix,
+    sister_list:       :sister_list,
+    total_runtime:     :total_runtime,
+    P:                 :proxy_user,
+    node_exclusive:    :node_exclusive,
+    exec_gpus:         :exec_gpus,
+    exec_mics:         :exec_mics,
+    J:                 :job_id,
+    pagg:              :pagg_id,
+    system_start_time: :system_start_time,
+    gpu_flags:         :gpu_flags,
+
+    # Additional queue attribute names
+
+    # Additional server attribute names
+
+    # Additional node attribute names
+  }
+end