ood_core 0.11.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf5cfe29bd0770daa8404169e04fcc8fcdc9a89b88f83bbfbc8675040b119ccf
-  data.tar.gz: c6082f2c7b751c0b7f247dcc790ae35bc1a372830b6f7c529d1b5574eed714ee
+  metadata.gz: 3296708d7bc47f3379a9e4a6c845d3f25c5ccefb599f4b92406d9dffdaef220b
+  data.tar.gz: b6af9e90b67bc9a7a52203808d849d8800336b30b09bdb8ed204526d01bc92e9
 SHA512:
-  metadata.gz: 30c82f37cf6c974c04a3d8c9bc9da21e47014b7aecfa03424575196e42aa0ebebb89aba6717e73073a1ed3d963d8391fa76da263a4e3a7e4dc4250a2cb32f830
-  data.tar.gz: 9c7be268d29f4dd6c9cec57ce783e7eb0777c76e3df7025062e77868985a61e410cd24ec7d39c2a4e91a039d85a85c41fad75e28a408c8e00940597e5a1fb1ff
+  metadata.gz: 623ac6e6f8081d68a3e925d1150c9f20a0f613ccfb6837519d1b95d04533a72caa403c54327aad85dcea9c0694cc23941f40307d942623c095f53fed7fc32026
+  data.tar.gz: 0d785a9ade36b2f6f62f9ae55672091346aa4fb76bf358e6c00d4bc007623b8d1798813474665fc7b4d850d89e041fae5c2fefc9719fbe9f53a161a76127eaad

data/CHANGELOG.md

@@ -6,12 +6,40 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.13.0] - 2020-08-10
+### Added
+- CloudyCluster CCQ Adapter
+
+## [0.12.0] - 2020-08-05
+### Added
+- qos option to Slurm and Torque [#205](https://github.com/OSC/ood_core/pull/205)
+- native hash returned in qstat for SGE adapter [#198](https://github.com/OSC/ood_core/pull/198)
+- option for specifying `submit_host` to submit jobs via ssh on other host [#204](https://github.com/OSC/ood_core/pull/204)
+
+### Fixed
+- SGE handle milliseconds instead of seconds when milliseconds used [#206](https://github.com/OSC/ood_core/issues/206)
+- Torque's native "hash" for job submission now handles env vars values with spaces [#202](https://github.com/OSC/ood_core/pull/202)
+
+## [0.11.4] - 2020-05-27
+### Fixed
+- Environment exports in SLURM while implementing [#158](https://github.com/OSC/ood_core/issues/158)
+  and [#109](https://github.com/OSC/ood_core/issues/109) in [#163](https://github.com/OSC/ood_core/pull/163)
+
+## [0.11.3] - 2020-05-11
+### Fixed
+- LinuxhHost Adapter to work with any login shell ([#188](https://github.com/OSC/ood_core/pull/188))
+- LinuxhHost Adapter needs to display long lines in pstree to successfully parse
+  output ([#188](https://github.com/OSC/ood_core/pull/188))
+
+## [0.11.2] - 2020-04-23
+### Fixed
+- fix signature of `LinuxHost#info_where_owner`
 
-## [0.11.1] - 2012-03-18
+## [0.11.1] - 2020-03-18
 ### Changed
 - Only the version changed. Had to republish to rubygems.org
 
-## [0.11.0] - 2012-03-18
+## [0.11.0] - 2020-03-18
 ### Added
 - Added directive prefixes to each adapter (e.g. `#QSUB`) ([#161](https://github.com/OSC/ood_core/issues/161))
 - LHA supports `submit_host` field in native ([#164](https://github.com/OSC/ood_core/issues/164))
@@ -219,7 +247,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.11.1...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.13.0...HEAD
+[0.13.0]: https://github.com/OSC/ood_core/compare/v0.12.0...v0.13.0
+[0.12.0]: https://github.com/OSC/ood_core/compare/v0.11.4...v0.12.0
+[0.11.4]: https://github.com/OSC/ood_core/compare/v0.11.3...v0.11.4
+[0.11.3]: https://github.com/OSC/ood_core/compare/v0.11.2...v0.11.3
+[0.11.2]: https://github.com/OSC/ood_core/compare/v0.11.1...v0.11.2
 [0.11.1]: https://github.com/OSC/ood_core/compare/v0.11.0...v0.11.1
 [0.11.0]: https://github.com/OSC/ood_core/compare/v0.10.0...v0.11.0
 [0.10.0]: https://github.com/OSC/ood_core/compare/v0.9.3...v0.10.0

data/README.md

@@ -4,12 +4,13 @@
 ![GitHub Release](https://img.shields.io/github/release/osc/ood_core.svg)
 ![GitHub License](https://img.shields.io/github/license/osc/ood_core.svg)
 
-Welcome to your new gem! In this directory, you'll find the files you need to
-be able to package up your Ruby library into a gem. Put your Ruby code in the
-file `lib/ood_core`. To experiment with that code, run `bin/console` for an
-interactive prompt.
+- Website: http://openondemand.org/
+- Website repo with JOSS publication: https://github.com/OSC/Open-OnDemand
+- Documentation: https://osc.github.io/ood-documentation/master/
+- Main code repo: https://github.com/OSC/ondemand
+- Core library repo: https://github.com/OSC/ood_core
 
-TODO: Delete this and the text above, and describe your gem
+OnDemand core library with adapters for each batch scheduler.
 
 ## Installation
 

data/lib/ood_core/job/adapters/ccq.rb

@@ -0,0 +1,267 @@
+require "ood_core/job/adapters/helper"
+require "tempfile"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      # Build the Cloudy Cluster adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :image (nil) The default VM image to use
+      # @option config [Object] :cloud (gcp) The cloud provider being used [gcp,aws]
+      # @option config [Object] :scheduler (nil) The name of the scheduler to use
+      # @option config [Object] :sge_root (nil) Path to SGE root, note that
+      # @option config [#to_h] :bin (nil) Path to CC client binaries
+      # @option config [#to_h] :bin_overrides ({}) Optional overrides to CC client executables
+      def self.build_ccq(config)
+        Adapters::CCQ.new(config.to_h.symbolize_keys)
+      end
+    end
+
+    module Adapters
+
+      class PromptError < StandardError; end
+
+      class CCQ < Adapter
+        using Refinements::ArrayExtensions
+
+        attr_reader :image, :cloud, :scheduler, :bin, :bin_overrides, :jobid_regex
+
+        def initialize(config)
+          @image = config.fetch(:image, nil)
+          @cloud = config.fetch(:cloud, gcp_provider)
+          @scheduler = config.fetch(:scheduler, nil)
+          @bin = config.fetch(:bin, '/opt/CloudyCluster/srv/CCQ')
+          @bin_overrides = config.fetch(:bin_overrides, {})
+          @jobid_regex = config.fetch(:jobid_regex, "job id is: (?<job_id>\\d+) you")
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @param script [Script] script object that describes the script and
+        #   attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] not used
+        # @param afterok [#to_s, Array<#to_s>] not used
+        # @param afternotok [#to_s, Array<#to_s>] not used
+        # @param afterany [#to_s, Array<#to_s>] not used
+        # @return [String] the job id returned after successfully submitting a
+        #   job
+        # @see Adapter#submit
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          script_file = make_script_file(script.content)
+          args = []
+
+          # cluster configuration args
+          args.concat ["-s", scheduler] unless scheduler.nil?
+          args.concat [image_arg, image] unless image.nil?
+
+          args.concat ["-o", script.output_path.to_s] unless script.output_path.nil?
+          args.concat ["-e", script.error_path.to_s] unless script.error_path.nil?
+          args.concat ["-tl", seconds_to_duration(script.wall_time)] unless script.wall_time.nil?
+          args.concat ["-js", script_file.path.to_s]
+
+          args.concat script.native if script.native
+
+          output = call("ccqsub", args: args)
+          parse_job_id_from_ccqsub(output)
+        ensure
+          script_file.close
+        end
+
+        # Retrieve info for all jobs from the resource manager
+        # @return [Array<Info>] information describing submitted jobs
+        def info_all(attrs: nil)
+          args = []
+          args.concat ["-s", scheduler] unless scheduler.nil?
+
+          stat_output = call("ccqstat", args: args)
+          info_from_ccqstat(stat_output)
+        end
+
+        # Retrieve job info from the resource manager
+        # @param id [#to_s] the id of the job
+        # @return [Info] information describing submitted job
+        def info(id)
+          args = []
+          args.concat ["-s", scheduler] unless scheduler.nil?
+          args.concat ["-ji", id]
+
+          stat_output = call("ccqstat", args: args)
+
+          # WARNING: code path differs here than info_all because the output
+          # from ccqstat -ji $JOBID is much more data than just the 4
+          # columns that ccqstat gives.
+          info_from_ccqstat_extended(stat_output)
+        end
+
+        # Retrieve job status from resource manager
+        # @param id [#to_s] the id of the job
+        # @return [Status] status of job
+        # @see Adapter#status
+        def status(id)
+          info(id).status
+        end
+
+        # This adapter does not implement hold and will always raise
+        #   an exception.
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] always
+        # @return [void]
+        def hold(_)
+          raise NotImplementedError, "subclass did not define #hold"
+        end
+
+        # This adapter does not implement release and will always raise
+        #   an exception.
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] always
+        # @return [void]
+        def release(_)
+          raise NotImplementedError, "subclass did not define #release"
+        end
+
+        # Delete the submitted job
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def delete(id)
+          call("ccqdel", args: [id])
+        end
+
+        def directive_prefix
+          '#CC'
+        end
+
+        private
+
+        # Mapping of state codes
+        STATE_MAP =
+        {
+          'Error'         => :suspended,  # not running, but infrastructure still possibly exists
+          'CreatingCG'    => :queued,     # creating control group
+          'Pending'       => :queued,     # in queue
+          'Submitted'     => :queued,     #
+          'Provisioning'  => :queued,     # node is being provisioned
+          'Running'       => :running,    #
+          'Completed'     => :completed,  #
+        }.freeze
+
+        def gcp_provider
+          'gcp'
+        end
+
+        def aws_provider
+          'aws'
+        end
+
+        def image_arg
+          if cloud == gcp_provider
+            '-gcpgi'
+          else
+            '-awsami'
+          end
+        end
+
+        def call(cmd, args: [], env: {}, stdin: "")
+          cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
+          args = args.map(&:to_s)
+          env = env.to_h
+          o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
+          s.success? ? o : interpret_and_raise(e, cmd)
+        end
+
+        # helper function to interpret an error the command had given and
+        # raise a different error.
+        def interpret_and_raise(error, command)
+          # a special case with CCQ that prompts the user for username & password
+          # so let's be helpful and tell the user what to do.
+          if error.end_with?("EOFError: EOF when reading a line\n")
+            raise(
+              PromptError,
+              "The #{command} command was prompted. You need to generate the certificate " + 
+              "manually in a shell by running 'ccqstat'\nand entering your username/password"
+            )
+          else
+            raise(JobAdapterError, e.message)
+          end
+        end
+
+        # Convert seconds to duration
+        def seconds_to_duration(seconds)
+          format("%02d:%02d:%02d", seconds / 3600, seconds / 60 % 60, seconds % 60)
+        end
+
+        # helper to make a script file. We can't pipe it into ccq so we have to
+        # write a file.
+        def make_script_file(content)
+          file = Tempfile.new(tmp_file_name)
+          file.write(content.to_s)
+          file.flush
+          file
+        end
+
+        def tmp_file_name
+          'ccq_ood_script_'
+        end
+
+        def parse_job_id_from_ccqsub(output)
+          match_data = /#{jobid_regex}/.match(output)
+          # match_data could be nil, OR re-configured jobid_regex could be looking for a different named group
+          job_id = match_data&.named_captures&.fetch('job_id', nil)
+          throw JobAdapterError.new "Could not extract job id out of ccqsub output '#{output}'" if job_id.nil?
+          job_id
+        end
+
+        # parse an Ood::Job::Info object from extended ccqstat output
+        def info_from_ccqstat_extended(data)
+          raw = extended_data_to_hash(data)
+          data_hash = { native: raw }
+          data_hash[:status] = get_state(raw['status'])
+          data_hash[:id] = raw['name']
+          data_hash[:job_name] = raw['jobName']
+          data_hash[:job_owner] = raw['userName']
+          data_hash[:submit_host] = raw['submitHostInstanceId']
+          data_hash[:dispatch_time] = raw['startTime'].to_i
+          data_hash[:submission_time] = raw['dateSubmitted'].to_i
+          data_hash[:queue_name] = raw['criteriaPriority']
+
+          Info.new(data_hash)
+        end
+
+        # extended data is just lines of 'key: value' value, so parse
+        # it and stick it all in a hash.
+        def extended_data_to_hash(data)
+          Hash[data.to_s.scan(/(\w+): (\S+)/)]
+        end
+
+        def info_from_ccqstat(data)
+          infos = []
+
+          data.to_s.each_line do |line|
+            words = line.split(/\s/).reject(&:empty?)
+            next if !words.empty? && words[0] == "Id" # just skip the header
+
+            infos << Info.new(line_to_hash(words)) if words.size == 5
+          end
+
+          infos
+        end
+
+        def line_to_hash(words)
+          return unless words.size == 5
+
+          data_hash = {}
+          data_hash[:id] = words[0]
+          data_hash[:job_name] = words[1]
+          data_hash[:job_owner] = words[2]
+          data_hash[:status] = get_state(words[4])
+
+          data_hash
+        end
+
+        def get_state(state)
+          STATE_MAP.fetch(state, :undetermined)
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/job/adapters/helper.rb

@@ -12,7 +12,26 @@ module OodCore
         def self.bin_path(cmd, bin_default, bin_overrides)
           bin_overrides.fetch(cmd.to_s) { Pathname.new(bin_default.to_s).join(cmd.to_s).to_s }
         end
+        
+        # Gets a command that submits command on another host via ssh
+        # @param submit_host [String] where to submit the command
+        # @param cmd [String] the desired command to execute on another host
+        # @param cmd_args [Array] arguments to the command specified above
+        # @param strict_host_checking [Bool] whether to use strict_host_checking
+        # @param env [Hash] env variables to be set w/ssh
+        #
+        # @return cmd [String] command wrapped in ssh if submit_host is present
+        # @return args [Array] command arguments including ssh_flags and original command
+        def self.ssh_wrap(submit_host, cmd, cmd_args, strict_host_checking = true, env = {})
+          return cmd, cmd_args if submit_host.to_s.empty?
+
+          check_host = strict_host_checking ? "yes" : "no"
+          args = ['-o', 'BatchMode=yes', '-o', 'UserKnownHostsFile=/dev/null', '-o', "StrictHostKeyChecking=#{check_host}", "#{submit_host}"]
+          env.each{|key, value| args.push("export #{key}=#{value};")}
+
+          return 'ssh', args + [cmd] + cmd_args
+        end
       end
     end
   end
-end
+end

data/lib/ood_core/job/adapters/linux_host.rb

@@ -106,7 +106,7 @@ module OodCore
         # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
-        def info_where_owner(owner: nil, attrs: nil)
+        def info_where_owner(_, attrs: nil)
           info_all
         end
 

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -57,7 +57,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   # @param hostname [#to_s] The hostname to submit the work to
   # @param script [OodCore::Job::Script] The script object defining the work
   def start_remote_session(script)
-    cmd = ssh_cmd(submit_host(script))
+    cmd = ssh_cmd(submit_host(script), ['/usr/bin/env', 'bash'])
 
     session_name = unique_session_name
     output = call(*cmd, stdin: wrapped_script(script, session_name))
@@ -67,13 +67,13 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   end
 
   def stop_remote_session(session_name, hostname)
-    cmd = ssh_cmd(hostname)
+    cmd = ssh_cmd(hostname, ['/usr/bin/env', 'bash'])
 
     kill_cmd = <<~SCRIPT
     # Get the tmux pane PID for the target session
     pane_pid=$(tmux list-panes -aF '\#{session_name} \#{pane_pid}' | grep '#{session_name}' | cut -f 2 -d ' ')
     # Find the Singularity sinit PID child of the pane process
-    pane_sinit_pid=$(pstree -p "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
+    pane_sinit_pid=$(pstree -p -l "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
     # Kill sinit which stops both Singularity-based processes and the tmux session
     kill "$pane_sinit_pid"
     SCRIPT
@@ -116,19 +116,23 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     s.success? ? o : raise(Error, e)
   end
 
-  # The SSH invocation to send a command
+  # The full command to ssh into the destination host and execute the command.
+  # SSH options include:
   # -t Force pseudo-terminal allocation (required to allow tmux to run)
   # -o BatchMode=yes (set mode to be non-interactive)
   # if ! strict_host_checking
   # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
   # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
-  def ssh_cmd(destination_host)
+  #
+  # @param destination_host [#to_s] the destination host you wish to ssh into
+  # @param cmd [Array<#to_s>] the command to be executed on the destination host
+  def ssh_cmd(destination_host, cmd)
     if strict_host_checking
       [
         'ssh', '-t',
         '-o', 'BatchMode=yes',
         "#{username}@#{destination_host}"
-      ]
+      ].concat(cmd)
     else
       [
         'ssh', '-t',
@@ -136,7 +140,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         '-o', 'UserKnownHostsFile=/dev/null',
         '-o', 'StrictHostKeyChecking=no',
         "#{username}@#{destination_host}"
-      ]
+      ].concat(cmd)
     end
   end
 
@@ -170,6 +174,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'session_name' => session_name,
         'singularity_bin' => singularity_bin,
         'singularity_image' => singularity_image(script.native),
+        'ssh_hosts' => ssh_hosts,
         'tmux_bin' => tmux_bin,
       }.each{
         |key, value| bnd.local_variable_set(key, value)
@@ -245,7 +250,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
       ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
     )
     keys = [:session_name, :session_created, :session_pid]
-    cmd = ssh_cmd(destination_host) + ['tmux', 'list-panes', '-aF', format_str]
+    cmd = ssh_cmd(destination_host, ['tmux', 'list-panes', '-aF', format_str])
 
     call(*cmd).split(
       "\n"