ood_core 0.11.3 → 0.15.0

data/lib/ood_core/job/adapters/sge/batch.rb

@@ -15,7 +15,7 @@ end
 class OodCore::Job::Adapters::Sge::Batch
   using OodCore::Refinements::HashExtensions
 
-  attr_reader :bin, :bin_overrides, :conf, :cluster, :helper
+  attr_reader :bin, :bin_overrides, :conf, :cluster, :helper, :submit_host, :strict_host_checking
   
   require "ood_core/job/adapters/sge/qstat_xml_j_r_listener"
   require "ood_core/job/adapters/sge/qstat_xml_r_listener"
@@ -36,6 +36,8 @@ class OodCore::Job::Adapters::Sge::Batch
     @bin              = Pathname.new(config.fetch(:bin, nil).to_s)
     @sge_root         = Pathname.new(config[:sge_root] || ENV['SGE_ROOT'] || "/var/lib/gridengine")
     @bin_overrides    = config.fetch(:bin_overrides, {})
+    @submit_host      = config.fetch(:submit_host, "")
+    @strict_host_checking = config.fetch(:strict_host_checking, true)
 
     # FIXME: hack as this affects env of the process!
     ENV['SGE_ROOT'] = @sge_root.to_s
@@ -62,7 +64,7 @@ class OodCore::Job::Adapters::Sge::Batch
   def get_all(owner: nil)
     listener = QstatXmlRListener.new
     argv = ['qstat', '-r', '-xml']
-    argv += ['-u', owner] unless owner.nil?
+    argv.concat ['-u', owner] unless owner.nil?
     REXML::Parsers::StreamParser.new(call(*argv), listener).parse
 
     listener.parsed_jobs.map{
@@ -166,11 +168,10 @@ class OodCore::Job::Adapters::Sge::Batch
   # Call a forked SGE command for a given batch server
   def call(cmd, *args, env: {}, stdin: "", chdir: nil)
     cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
-    args = args.map(&:to_s)
-
     env = env.to_h.each_with_object({}) { |(k, v), h| h[k.to_s] = v.to_s }
+    cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking, env)
     chdir ||= "."
-    o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s, chdir: chdir.to_s)
+    o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin.to_s, chdir: chdir.to_s)
     s.success? ? o : raise(Error, e)
   end
 

data/lib/ood_core/job/adapters/sge/helper.rb

@@ -17,38 +17,38 @@ class OodCore::Job::Adapters::Sge::Helper
     raise_error_on_unsupported_args(script, after: after, afterok: afterok, afternotok: afternotok, afterany: afterany)
 
     args = []
-    args += ['-h'] if script.submit_as_hold
-    args += ['-r', 'yes'] if script.rerunnable
-    script.job_environment.each_pair {|k, v| args += ['-v', "#{k.to_s}=#{v.to_s}"]} unless script.job_environment.nil?
-    args += ["-V"] if script.copy_environment?
+    args.concat ['-h'] if script.submit_as_hold
+    args.concat ['-r', 'yes'] if script.rerunnable
+    script.job_environment.each_pair {|k, v| args.concat ['-v', "#{k.to_s}=#{v.to_s}"]} unless script.job_environment.nil?
+    args.concat ["-V"] if script.copy_environment?
 
     if script.workdir
-      args += ['-wd', script.workdir]
+      args.concat ['-wd', script.workdir]
     elsif ! script_contains_wd_directive?(script.content)
-      args += ['-cwd']
+      args.concat ['-cwd']
     end
 
     on_event_email = []
     on_event_email << 'b' if script.email_on_started  # beginning
     on_event_email << 'ea' if script.email_on_terminated  # end, aborted
 
-    args += ['-M', script.email.first, '-m', on_event_email.join] if script.email && ! on_event_email.empty?
+    args.concat ['-M', script.email.first, '-m', on_event_email.join] if script.email && ! on_event_email.empty?
 
     afterok  = Array(afterok).map(&:to_s)
-    args += ['-hold_jid_ad', afterok.join(',')] unless afterok.empty?
+    args.concat ['-hold_jid_ad', afterok.join(',')] unless afterok.empty?
 
     # ignoring email_on_started
-    args += ['-N', script.job_name] unless script.job_name.nil?
-    args += ['-e', script.error_path] unless script.error_path.nil?
-    args += ['-o', script.output_path] unless script.output_path.nil?
-    args += ['-ar', script.reservation_id] unless script.reservation_id.nil?
-    args += ['-q', script.queue_name] unless script.queue_name.nil?
-    args += ['-p', script.priority] unless script.priority.nil?
-    args += ['-a', script.start_time.strftime('%C%y%m%d%H%M.%S')] unless script.start_time.nil?
-    args += ['-l', "h_rt=" + seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
-    args += ['-P', script.accounting_id] unless script.accounting_id.nil?
-    args += ['-t', script.job_array_request] unless script.job_array_request.nil?
-    args += Array.wrap(script.native) if script.native
+    args.concat ['-N', script.job_name] unless script.job_name.nil?
+    args.concat ['-e', script.error_path] unless script.error_path.nil?
+    args.concat ['-o', script.output_path] unless script.output_path.nil?
+    args.concat ['-ar', script.reservation_id] unless script.reservation_id.nil?
+    args.concat ['-q', script.queue_name] unless script.queue_name.nil?
+    args.concat ['-p', script.priority] unless script.priority.nil?
+    args.concat ['-a', script.start_time.strftime('%C%y%m%d%H%M.%S')] unless script.start_time.nil?
+    args.concat ['-l', "h_rt=" + seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
+    args.concat ['-P', script.accounting_id] unless script.accounting_id.nil?
+    args.concat ['-t', script.job_array_request] unless script.job_array_request.nil?
+    args.concat Array.wrap(script.native) if script.native
 
     args
   end

data/lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb

@@ -28,10 +28,13 @@ class QstatXmlJRListener
       :tasks => [],
       :status => :queued,
       :procs => 1,
-      :native => {}  # TODO: improve native attribute reporting
+      :native => {
+        :ST_name => ''
+      } 
     }
     @current_text = nil
     @current_request = nil
+    @processing_JB_stdout_path_list = false
 
     @processing_job_array_spec = false
     @adding_slots = false
@@ -42,6 +45,7 @@ class QstatXmlJRListener
       step: 1,  # Step can have a default of 1
     }
     @running_tasks = []
+    @native_tags = ['JB_job_number', 'JB_job_name', 'JB_version', 'JB_project', 'JB_exec_file', 'JB_script_file', 'JB_script_size', 'JB_submission_time', 'JB_execution_time', 'JB_deadline', 'JB_owner', 'JB_uid', 'JB_group', 'JB_gid', 'JB_account', 'JB_cwd', 'JB_notify', 'JB_type', 'JB_reserve', 'JB_priority', 'JB_jobshare', 'JB_verify', 'JB_checkpoint_attr', 'JB_checkpoint_interval', 'JB_restart']
   end
 
   def tag_start(name, attrs)
@@ -50,10 +54,17 @@ class QstatXmlJRListener
       toggle_processing_array_spec
     when 'JB_pe_range'
       toggle_adding_slots
+    when 'JB_stdout_path_list'
+      @processing_JB_stdout_path_list = true
     end
   end
 
   def tag_end(name)
+    #Add to native hash if in native_tags
+    if (@native_tags.include?(name))
+      @parsed_job[:native][:"#{name}"] = @current_text
+    end
+
     case name
     when 'JB_ja_tasks'
       end_JB_ja_tasks
@@ -92,6 +103,10 @@ class QstatXmlJRListener
       toggle_processing_array_spec
     when 'JB_pe_range'
       toggle_adding_slots
+    when 'PN_path' 
+      end_PN_path
+    when 'ST_name'
+      end_ST_name
     end
   end
 
@@ -118,7 +133,7 @@ class QstatXmlJRListener
   end
 
   def end_JB_submission_time
-    @parsed_job[:submission_time] = @current_text.to_i
+    @parsed_job[:submission_time] = ms_to_seconds(@current_text.to_i)
   end
 
   def end_JB_ja_tasks
@@ -127,7 +142,7 @@ class QstatXmlJRListener
 
   def end_JAT_start_time
     @parsed_job[:status] = :running
-    @parsed_job[:dispatch_time] = @current_text.to_i
+    @parsed_job[:dispatch_time] = ms_to_seconds(@current_text.to_i)
     @parsed_job[:wallclock_time] = Time.now.to_i - @parsed_job[:dispatch_time]
   end
 
@@ -153,6 +168,15 @@ class QstatXmlJRListener
     @running_tasks << @current_text
   end
 
+  def end_PN_path
+    @parsed_job[:native][:PN_path] = @current_text if @processing_JB_stdout_path_list
+    @processing_JB_stdout_path_list = false
+  end
+
+  def end_ST_name
+    @parsed_job[:native][:ST_name] = @parsed_job[:native][:ST_name] + @current_text + ' '
+  end
+
   def set_job_array_piece(key)
     @job_array_spec[key] = @current_text if @processing_job_array_spec
   end
@@ -201,5 +225,12 @@ class QstatXmlJRListener
   def set_slots
     @parsed_job[:procs] = @current_text.to_i
   end
-end
 
+  private
+
+  # Some Grid Engines (like UGE) use milliseconds were others use
+  # seconds past the epoch.
+  def ms_to_seconds(raw)
+    raw.digits.length >= 13 ? raw / 1000 : raw
+  end
+end

data/lib/ood_core/job/adapters/sge/qstat_xml_r_listener.rb

@@ -24,21 +24,32 @@ class QstatXmlRListener
     @parsed_jobs = []
     @current_job = {
       :tasks => [],
-      :native => {}  # TODO: improve native reporting
+      :native => {
+        :ST_name => ''
+      } 
     }
     @current_text = nil
+    @processing_JB_stdout_path_list = false
 
     @current_request = nil
+    @native_tags = ['JB_job_number', 'JB_job_name', 'JB_version', 'JB_project', 'JB_exec_file', 'JB_script_file', 'JB_script_size', 'JB_submission_time', 'JB_execution_time', 'JB_deadline', 'JB_owner', 'JB_uid', 'JB_group', 'JB_gid', 'JB_account', 'JB_cwd', 'JB_notify', 'JB_type', 'JB_reserve', 'JB_priority', 'JB_jobshare', 'JB_verify', 'JB_checkpoint_attr', 'JB_checkpoint_interval', 'JB_restart']
   end
 
   def tag_start(name, attributes)
     case name
     when 'hard_request'
       start_hard_request(attributes)
+    when "JB_stdout_path_list"
+      @processing_JB_stdout_path_list = true
     end
   end
 
   def tag_end(name)
+    #Add text if in native_tags
+    if (@native_tags.include?(name))
+      @current_job[:native][:"#{name}"] = @current_text
+    end
+    
     case name
     when 'job_list'
       end_job_list
@@ -64,6 +75,10 @@ class QstatXmlRListener
       end_hard_request
     when 'tasks'
       add_child_tasks
+    when 'PN_path'
+      end_PN_path
+    when 'ST_name'
+      end_ST_name
     end
   end
 
@@ -130,6 +145,15 @@ class QstatXmlRListener
     end
   end
 
+  def end_PN_path
+    @current_job[:native][:PN_path] = @current_text if @processing_JB_stdout_path_list
+    @processing_JB_stdout_path_list = false
+  end
+
+  def end_ST_name
+    @current_job[:native][:ST_name] = @current_job[:native][:ST_name] + @current_text + ' '
+  end
+
   # Store a completed job and reset current_job for the next pass
   def end_job_list
     @parsed_jobs << @current_job
@@ -145,4 +169,3 @@ class QstatXmlRListener
     }
   end
 end
-

data/lib/ood_core/job/adapters/slurm.rb

@@ -14,13 +14,17 @@ module OodCore
       # @option config [Object] :conf (nil) Path to the slurm conf
       # @option config [Object] :bin (nil) Path to slurm client binaries
       # @option config [#to_h] :bin_overrides ({}) Optional overrides to Slurm client executables
+      # @option config [Object] :submit_host ("") Submit job on login node via ssh
+      # @option config [Object] :strict_host_checking (true) Whether to use strict host checking when ssh to submit_host
       def self.build_slurm(config)
         c = config.to_h.symbolize_keys
-        cluster = c.fetch(:cluster, nil)
-        conf    = c.fetch(:conf, nil)
-        bin     = c.fetch(:bin, nil)
-        bin_overrides = c.fetch(:bin_overrides, {})
-        slurm = Adapters::Slurm::Batch.new(cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides)
+        cluster              = c.fetch(:cluster, nil)
+        conf                 = c.fetch(:conf, nil)
+        bin                  = c.fetch(:bin, nil)
+        bin_overrides        = c.fetch(:bin_overrides, {})
+        submit_host          = c.fetch(:submit_host, "")
+        strict_host_checking = c.fetch(:strict_host_checking, true)
+        slurm = Adapters::Slurm::Batch.new(cluster: cluster, conf: conf, bin: bin, bin_overrides: bin_overrides, submit_host: submit_host, strict_host_checking: strict_host_checking)
         Adapters::Slurm.new(slurm: slurm)
       end
     end
@@ -62,18 +66,36 @@ module OodCore
           # @return Hash<String, String>
           attr_reader :bin_overrides
 
+          # The login node where the job is submitted via ssh
+          # @example owens.osc.edu
+          # @return [String] The login node
+          attr_reader :submit_host
+
+          # Wheter to use strict host checking when ssh to submit_host
+          # @example false
+          # @return [Bool]; true if empty
+          attr_reader :strict_host_checking
+
           # The root exception class that all Slurm-specific exceptions inherit
           # from
           class Error < StandardError; end
 
+          # An error indicating the slurm command timed out
+          class SlurmTimeoutError < Error; end
+
           # @param cluster [#to_s, nil] the cluster name
           # @param conf [#to_s, nil] path to the slurm conf
           # @param bin [#to_s] path to slurm installation binaries
-          def initialize(cluster: nil, bin: nil, conf: nil, bin_overrides: {})
-            @cluster = cluster && cluster.to_s
-            @conf    = conf    && Pathname.new(conf.to_s)
-            @bin     = Pathname.new(bin.to_s)
-            @bin_overrides = bin_overrides
+          # @param bin_overrides [#to_h] a hash of bin ovverides to be used in job
+          # @param submit_host [#to_s] Submits the job on a login node via ssh
+          # @param strict_host_checking [Bool] Whether to use strict host checking when ssh to submit_host
+          def initialize(cluster: nil, bin: nil, conf: nil, bin_overrides: {}, submit_host: "", strict_host_checking: true)
+            @cluster              = cluster && cluster.to_s
+            @conf                 = conf    && Pathname.new(conf.to_s)
+            @bin                  = Pathname.new(bin.to_s)
+            @bin_overrides        = bin_overrides
+            @submit_host          = submit_host.to_s
+            @strict_host_checking = strict_host_checking
           end
 
           # Get a list of hashes detailing each of the jobs on the batch server
@@ -128,6 +150,9 @@ module OodCore
               end
               jobs
             end
+          rescue SlurmTimeoutError
+            # TODO: could use a log entry here
+            return [{ id: id, state: 'undetermined' }]
           end
 
           def squeue_fields(attrs)
@@ -148,9 +173,9 @@ module OodCore
           #TODO: write some barebones test for this? like 2 options and id or no id
           def squeue_args(id: "", owner: nil, options: [])
             args  = ["--all", "--states=all", "--noconvert"]
-            args += ["-o", "#{RECORD_SEPARATOR}#{options.join(UNIT_SEPARATOR)}"]
-            args += ["-u", owner.to_s] unless owner.to_s.empty?
-            args += ["-j", id.to_s] unless id.to_s.empty?
+            args.concat ["-o", "#{RECORD_SEPARATOR}#{options.join(UNIT_SEPARATOR)}"]
+            args.concat ["-u", owner.to_s] unless owner.to_s.empty?
+            args.concat ["-j", id.to_s] unless id.to_s.empty?
             args
           end
 
@@ -275,12 +300,27 @@ module OodCore
             # Call a forked Slurm command for a given cluster
             def call(cmd, *args, env: {}, stdin: "")
               cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
+
               args  = args.map(&:to_s)
-              args += ["-M", cluster] if cluster
+              args.concat ["-M", cluster] if cluster
+
               env = env.to_h
               env["SLURM_CONF"] = conf.to_s if conf
-              o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
-              s.success? ? o : raise(Error, e)
+
+              cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking)
+              o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin.to_s)
+              s.success? ? interpret_and_raise(o, e) : raise(Error, e)
+            end
+
+            # Helper function to raise an error based on the contents of stderr.
+            # Slurm exits 0 even when the command fails, so we need to interpret stderr
+            # to see if the command was actually successful.
+            def interpret_and_raise(stdout, stderr)
+              return stdout if stderr.empty?
+
+              raise SlurmTimeoutError, stderr if /^slurm_load_jobs error: Socket timed out/.match(stderr)
+
+              stdout
             end
 
             def squeue_attrs_for_info_attrs(attrs)
@@ -358,30 +398,31 @@ module OodCore
           # Set sbatch options
           args = []
           # ignore args, don't know how to do this for slurm
-          args += ["-H"] if script.submit_as_hold
-          args += (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
-          args += ["-D", script.workdir.to_s] unless script.workdir.nil?
-          args += ["--mail-user", script.email.join(",")] unless script.email.nil?
+          args.concat ["-H"] if script.submit_as_hold
+          args.concat (script.rerunnable ? ["--requeue"] : ["--no-requeue"]) unless script.rerunnable.nil?
+          args.concat ["-D", script.workdir.to_s] unless script.workdir.nil?
+          args.concat ["--mail-user", script.email.join(",")] unless script.email.nil?
           if script.email_on_started && script.email_on_terminated
-            args += ["--mail-type", "ALL"]
+            args.concat ["--mail-type", "ALL"]
           elsif script.email_on_started
-            args += ["--mail-type", "BEGIN"]
+            args.concat ["--mail-type", "BEGIN"]
           elsif script.email_on_terminated
-            args += ["--mail-type", "END"]
+            args.concat ["--mail-type", "END"]
           elsif script.email_on_started == false && script.email_on_terminated == false
-            args += ["--mail-type", "NONE"]
+            args.concat ["--mail-type", "NONE"]
           end
-          args += ["-J", script.job_name] unless script.job_name.nil?
-          args += ["-i", script.input_path] unless script.input_path.nil?
-          args += ["-o", script.output_path] unless script.output_path.nil?
-          args += ["-e", script.error_path] unless script.error_path.nil?
-          args += ["--reservation", script.reservation_id] unless script.reservation_id.nil?
-          args += ["-p", script.queue_name] unless script.queue_name.nil?
-          args += ["--priority", script.priority] unless script.priority.nil?
-          args += ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
-          args += ["-A", script.accounting_id] unless script.accounting_id.nil?
-          args += ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
-          args += ['-a', script.job_array_request] unless script.job_array_request.nil?
+          args.concat ["-J", script.job_name] unless script.job_name.nil?
+          args.concat ["-i", script.input_path] unless script.input_path.nil?
+          args.concat ["-o", script.output_path] unless script.output_path.nil?
+          args.concat ["-e", script.error_path] unless script.error_path.nil?
+          args.concat ["--reservation", script.reservation_id] unless script.reservation_id.nil?
+          args.concat ["-p", script.queue_name] unless script.queue_name.nil?
+          args.concat ["--priority", script.priority] unless script.priority.nil?
+          args.concat ["--begin", script.start_time.localtime.strftime("%C%y-%m-%dT%H:%M:%S")] unless script.start_time.nil?
+          args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
+          args.concat ["-t", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
+          args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
+          args.concat ['--qos', script.qos] unless script.qos.nil?
           # ignore nodes, don't know how to do this for slurm
 
           # Set dependencies
@@ -390,17 +431,14 @@ module OodCore
           depend << "afterok:#{afterok.join(":")}"       unless afterok.empty?
           depend << "afternotok:#{afternotok.join(":")}" unless afternotok.empty?
           depend << "afterany:#{afterany.join(":")}"     unless afterany.empty?
-          args += ["-d", depend.join(",")]               unless depend.empty?
+          args.concat ["-d", depend.join(",")]               unless depend.empty?
 
           # Set environment variables
           env = script.job_environment || {}
-          unless (script.job_environment.nil? || script.job_environment.empty?)
-            prefix = script.copy_environment? ? "ALL," : "NONE,"  # NONE if false or nil
-            args += ["--export", prefix + script.job_environment.keys.join(",")]
-          end
+          args.concat ["--export", export_arg(env, script.copy_environment?)]
 
           # Set native options
-          args += script.native if script.native
+          args.concat script.native if script.native
 
           # Set content
           content = if script.shell_path.nil?
@@ -630,6 +668,25 @@ module OodCore
 
             Info.new(**parent_task_hash)
           end
+
+
+          # we default to export NONE, but SLURM defaults to ALL.
+          # we do this bc SLURM setups a new environment, loading /etc/profile
+          # and all giving 'module' function (among other things shells give),
+          # where the PUN did not.
+          # --export=ALL export the PUN's environment.
+          def export_arg(env, copy_environment)
+            if !env.empty? && !copy_environment
+              env.keys.join(",")
+            elsif !env.empty? && copy_environment
+              "ALL," + env.keys.join(",")
+            elsif env.empty? && copy_environment
+              # only this option changes behaivor dramatically
+              "ALL"
+            else
+              "NONE"
+            end
+          end
       end
     end
   end