ood_core 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c668d456d8773bfa4a3af714f73a57d2f9396ee8
-  data.tar.gz: 14cdd12014850a9464505d7e34e5e4397826c6d9
+SHA256:
+  metadata.gz: bb944d43beb0aced99e13efb2ef10bf33f9666c705c50ca5ae1727751de43073
+  data.tar.gz: 6e3cd66160be3bbd63124d6f2ddc794bce4ae64e385977828faba5dfd28ff838
 SHA512:
-  metadata.gz: ebb2ea211dc272f03884faf0c2245e2f2b945851023ef5f5f26c1862514a0626a57c35c3be636d9287d1581d56dd18a3e6ff014e41837d63b7d05559605acd79
-  data.tar.gz: beb0f071c17c7632e9aae32ee243e3fd47480ff385d1d82cfe42e48652a6bbaaee3db7c224c7c2b4c8f91b11b11b8e5d9170b231f846e60ff9e6e1d2f03a0260
+  metadata.gz: 176e331a856c1e6958c444426d5c1b41aa881e90a69dca507b07f5463eb81355689e8391e0bf27823fc42a9484789f623ffd566b9d6c414c9cf741a7cafd1def
+  data.tar.gz: 15481101ad3120d3e8457612f2b8a8be4f1e268b38538b18b710f27887836f7a47eac3bf2e89d8f73745ae96ae78e21cd5ba5afefb4161cf95f435d6f2fdf001

data/.travis.yml

@@ -1,8 +1,8 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.4.1
-before_install: gem install bundler -v 1.7.8
+  - 2.5.5
+before_install: gem install bundler -v '~> 1.17'
 notifications:
   email:
     on_success: never

data/CHANGELOG.md

@@ -6,6 +6,53 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.12.0] - 2020-08-05
+### Added
+- qos option to Slurm and Torque [#205](https://github.com/OSC/ood_core/pull/205)
+- native hash returned in qstat for SGE adapter [#198](https://github.com/OSC/ood_core/pull/198)
+- option for specifying `submit_host` to submit jobs via ssh on other host [#204](https://github.com/OSC/ood_core/pull/204)
+
+### Fixed
+- SGE handle milliseconds instead of seconds when milliseconds used [#206](https://github.com/OSC/ood_core/issues/206)
+- Torque's native "hash" for job submission now handles env vars values with spaces [#202](https://github.com/OSC/ood_core/pull/202)
+
+## [0.11.4] - 2020-05-27
+### Fixed
+- Environment exports in SLURM while implementing [#158](https://github.com/OSC/ood_core/issues/158)
+  and [#109](https://github.com/OSC/ood_core/issues/109) in [#163](https://github.com/OSC/ood_core/pull/163)
+
+## [0.11.3] - 2020-05-11
+### Fixed
+- LinuxhHost Adapter to work with any login shell ([#188](https://github.com/OSC/ood_core/pull/188))
+- LinuxhHost Adapter needs to display long lines in pstree to successfully parse
+  output ([#188](https://github.com/OSC/ood_core/pull/188))
+
+## [0.11.2] - 2020-04-23
+### Fixed
+- fix signature of `LinuxHost#info_where_owner`
+
+## [0.11.1] - 2020-03-18
+### Changed
+- Only the version changed. Had to republish to rubygems.org
+
+## [0.11.0] - 2020-03-18
+### Added
+- Added directive prefixes to each adapter (e.g. `#QSUB`) ([#161](https://github.com/OSC/ood_core/issues/161))
+- LHA supports `submit_host` field in native ([#164](https://github.com/OSC/ood_core/issues/164))
+- Cluster files can be yaml or yml extensions ([#171](https://github.com/OSC/ood_core/issues/171))
+- Users can add a flag `OOD_JOB_NAME_ILLEGAL_CHARS` to sanitize job names ([#183](https://github.com/OSC/ood_core/issues/183)
+
+### Changed
+- Simplified job array parsing ([#144](https://github.com/OSC/ood_core/issues/144))
+
+### Fixed
+- Issue where environment variables were not properly exported to the job ([#158](https://github.com/OSC/ood_core/issues/158))
+- Parsing bad cluster files ([#150](https://github.com/OSC/ood_core/issues/150) and [#178](https://github.com/OSC/ood_core/issues/178))
+- netcat is no longer a hard dependency. Now lsof, python and bash can be used ([153](https://github.com/OSC/ood_core/issues/153))
+- GE crash when nil config file was given ([#175](https://github.com/OSC/ood_core/issues/175))
+- GE sometimes reported incorrect core count ([#168](https://github.com/OSC/ood_core/issues/168))
+
+
 ## [0.10.0] - 2019-11-05
 ### Added
 - Added an adapter for submitting work on Linux hosted systems without using a scheduler
@@ -196,7 +243,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.10.0...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.12.0...HEAD
+[0.12.0]: https://github.com/OSC/ood_core/compare/v0.11.4...v0.12.0
+[0.11.4]: https://github.com/OSC/ood_core/compare/v0.11.3...v0.11.4
+[0.11.3]: https://github.com/OSC/ood_core/compare/v0.11.2...v0.11.3
+[0.11.2]: https://github.com/OSC/ood_core/compare/v0.11.1...v0.11.2
+[0.11.1]: https://github.com/OSC/ood_core/compare/v0.11.0...v0.11.1
+[0.11.0]: https://github.com/OSC/ood_core/compare/v0.10.0...v0.11.0
 [0.10.0]: https://github.com/OSC/ood_core/compare/v0.9.3...v0.10.0
 [0.9.3]: https://github.com/OSC/ood_core/compare/v0.9.2...v0.9.3
 [0.9.2]: https://github.com/OSC/ood_core/compare/v0.9.1...v0.9.2

data/README.md

@@ -4,12 +4,13 @@
 ![GitHub Release](https://img.shields.io/github/release/osc/ood_core.svg)
 ![GitHub License](https://img.shields.io/github/license/osc/ood_core.svg)
 
-Welcome to your new gem! In this directory, you'll find the files you need to
-be able to package up your Ruby library into a gem. Put your Ruby code in the
-file `lib/ood_core`. To experiment with that code, run `bin/console` for an
-interactive prompt.
+- Website: http://openondemand.org/
+- Website repo with JOSS publication: https://github.com/OSC/Open-OnDemand
+- Documentation: https://osc.github.io/ood-documentation/master/
+- Main code repo: https://github.com/OSC/ondemand
+- Core library repo: https://github.com/OSC/ood_core
 
-TODO: Delete this and the text above, and describe your gem
+OnDemand core library with adapters for each batch scheduler.
 
 ## Installation
 

data/lib/ood_core.rb

@@ -2,6 +2,7 @@ require "ood_core/version"
 require "ood_core/errors"
 require "ood_core/cluster"
 require "ood_core/clusters"
+require "ood_core/invalid_cluster"
 
 # The main namespace for ood_core
 module OodCore

data/lib/ood_core/batch_connect/template.rb

@@ -117,11 +117,47 @@ module OodCore
                 }
                 export -f random_number
 
+                port_used_python() {
+                  python -c "import socket; socket.socket().connect(('$1',$2))" >/dev/null 2>&1
+                }
+
+                port_used_python3() {
+                  python3 -c "import socket; socket.socket().connect(('$1',$2))" >/dev/null 2>&1
+                }
+
+                port_used_nc(){
+                  nc -w 2 "$1" "$2" < /dev/null > /dev/null 2>&1
+                }
+
+                port_used_lsof(){
+                  lsof -i :"$2" >/dev/null 2>&1
+                }
+
+                port_used_bash(){
+                  local bash_supported=$(strings /bin/bash 2>/dev/null | grep tcp)
+                  if [ "$bash_supported" == "/dev/tcp/*/*" ]; then
+                    (: < /dev/tcp/$1/$2) >/dev/null 2>&1
+                  else
+                    return 127
+                  fi
+                }
+
                 # Check if port $1 is in use
                 port_used () {
                   local port="${1#*:}"
                   local host=$((expr "${1}" : '\\(.*\\):' || echo "localhost") | awk 'END{print $NF}')
-                  nc -w 2 "${host}" "${port}" < /dev/null &> /dev/null
+                  local port_strategies=(port_used_nc port_used_lsof port_used_bash port_used_python port_used_python3)
+
+                  for strategy in ${port_strategies[@]};
+                  do
+                    $strategy $host $port
+                    status=$?
+                    if [[ "$status" == "0" ]] || [[ "$status" == "1" ]]; then
+                      return $status
+                    fi
+                  done
+
+                  return 127
                 }
                 export -f port_used
 
@@ -143,8 +179,14 @@ module OodCore
                   local port="${1}"
                   local time="${2:-30}"
                   for ((i=1; i<=time*2; i++)); do
-                    if port_used "${port}"; then
+                    port_used "${port}"
+                    port_status=$?
+                    if [ "$port_status" == "0" ]; then
                       return 0
+                    elif [ "$port_status" == "127" ]; then
+                       echo "commands to find port were either not found or inaccessible."
+                       echo "command options are lsof, nc, bash's /dev/tcp, or python (or python3) with socket lib."
+                       return 127
                     fi
                     sleep 0.5
                   done

data/lib/ood_core/cluster.rb

@@ -28,6 +28,10 @@ module OodCore
     # @return [Hash] the acls configuration
     attr_reader :acls_config
 
+    # The errors encountered with configuring this cluster
+    # @return Array<String> the errors
+    attr_reader :errors
+
     # @param cluster [#to_h] the cluster object
     # @option cluster [#to_sym] :id The cluster id
     # @option cluster [#to_h] :metadata ({}) The cluster's metadata
@@ -39,6 +43,8 @@ module OodCore
     #   against
     # @option cluster [#to_h] :batch_connect ({}) Configuration for batch
     #   connect templates
+    # @option cluster [#to_a] :errors ([]) List of configuration errors
+    #
     def initialize(cluster)
       c = cluster.to_h.symbolize_keys
 
@@ -52,6 +58,9 @@ module OodCore
       @custom_config   = c.fetch(:custom, {})  .to_h.symbolize_keys
       @acls_config     = c.fetch(:acls, [])    .map(&:to_h)
       @batch_connect_config = c.fetch(:batch_connect, {}).to_h.symbolize_keys
+
+      # side affects from object creation and validation
+      @errors          = c.fetch(:errors, [])  .to_a
     end
 
     # Metadata that provides extra information about this cluster
@@ -159,6 +168,12 @@ module OodCore
       }
     end
 
+    # This cluster is always valid
+    # @return true
+    def valid?
+      return true
+    end
+
     private
       # Build acl adapter objects from array
       def build_acls(ary)

data/lib/ood_core/clusters.rb

@@ -21,16 +21,30 @@ module OodCore
         if config.file?
           if config.readable?
             CONFIG_VERSION.any? do |version|
-              YAML.safe_load(config.read).fetch(version, {}).each do |k, v|
-                clusters << Cluster.new(send("parse_#{version}", id: k, cluster: v))
+              begin
+                YAML.safe_load(config.read)&.fetch(version, {}).each do |k, v|
+                  clusters << Cluster.new(send("parse_#{version}", id: k, cluster: v))
+                end
+              rescue Psych::SyntaxError => e
+                clusters << InvalidCluster.new(
+                  id: config.basename(config.extname).to_s,
+                  errors: [ e.message.to_s ]
+                )
               end
             end
           end
         elsif config.directory?
-          Pathname.glob(config.join("*.yml")).select(&:file?).select(&:readable?).each do |p|
+          Pathname.glob([config.join("*.yml"), config.join("*.yaml")]).select(&:file?).select(&:readable?).each do |p|
             CONFIG_VERSION.any? do |version|
-              if cluster = YAML.safe_load(p.read).fetch(version, nil)
-                clusters << Cluster.new(send("parse_#{version}", id: p.basename(".yml").to_s, cluster: cluster))
+              begin
+                if cluster = YAML.safe_load(p.read)&.fetch(version, nil)
+                  clusters << Cluster.new(send("parse_#{version}", id: p.basename(p.extname()).to_s, cluster: cluster))
+                end
+              rescue Psych::SyntaxError => e
+                clusters << InvalidCluster.new(
+                  id: p.basename(p.extname).to_s,
+                  errors: [ e.message.to_s ]
+                )
               end
             end
           end

data/lib/ood_core/invalid_cluster.rb

@@ -0,0 +1,37 @@
+module OodCore
+  # A special case of an OodCore::Cluster where something went awry in the
+  # creation and it's invalid for some reason.  Users should only be able
+  # to rely on id and metadata.error_msg. All *allow? related functions
+  # false, meaning nothing is allowed.
+  class InvalidCluster < Cluster
+    # Jobs are not allowed
+    # @return false
+    def login_allow?
+      false
+    end
+
+    # Jobs are not allowed
+    # @return false
+    def job_allow?
+      false
+    end
+
+    # Custom features are not allowed
+    # @return false
+    def custom_allow?(_)
+      false
+    end
+
+    # This cluster is not allowed to be used
+    # @return false
+    def allow?
+      false
+    end
+
+    # This cluster is never valid
+    # @return false
+    def valid?
+      return false
+    end
+  end
+end

data/lib/ood_core/job/adapter.rb

@@ -36,7 +36,7 @@ module OodCore
       # Retrieve info for all jobs from the resource manager
       # @abstract Subclass is expected to implement {#info_all}
       # @raise [NotImplementedError] if subclass did not define {#info_all}
-      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided)
       #   This array specifies only attrs you want, in addition to id and status.
       #   If an array, the Info object that is returned to you is not guarenteed
       #   to have a value for any attr besides the ones specified and id and status.
@@ -51,7 +51,7 @@ module OodCore
       # Retrieve info for all jobs for a given owner or owners from the
       # resource manager
       # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
-      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided)
       #   This array specifies only attrs you want, in addition to id and status.
       #   If an array, the Info object that is returned to you is not guarenteed
       #   to have a value for any attr besides the ones specified and id and status.
@@ -69,7 +69,7 @@ module OodCore
       end
 
       # Iterate over each job Info object
-      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided)
       #   This array specifies only attrs you want, in addition to id and status.
       #   If an array, the Info object that is returned to you is not guarenteed
       #   to have a value for any attr besides the ones specified and id and status.
@@ -88,7 +88,7 @@ module OodCore
 
       # Iterate over each job Info object
       # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
-      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+      # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided)
       #   This array specifies only attrs you want, in addition to id and status.
       #   If an array, the Info object that is returned to you is not guarenteed
       #   to have a value for any attr besides the ones specified and id and status.
@@ -157,6 +157,37 @@ module OodCore
       def delete(id)
         raise NotImplementedError, "subclass did not define #delete"
       end
+
+      # Return the scheduler-specific directive prefix
+      #
+      # Examples of directive prefixes include #QSUB, #BSUB and allow placing what would
+      # otherwise be command line options inside the job launch script.
+      #
+      # The method should return nil if the adapter does not support prefixes
+      #
+      # @abstract Subclass is expected to implement {#directive_prefix}
+      # @raise [NotImplementedError] if subclass did not defined {#directive_prefix}
+      # @return [String]
+      def directive_prefix
+        raise NotImplementedError, "subclass did not define #directive_prefix"
+      end
+
+      # Replace illegal chars in job name with a dash
+      #
+      # @return [String] job name with dashes replacing illegal chars
+      def sanitize_job_name(job_name)
+        # escape ^ and omit -
+        chars = job_name_illegal_chars.to_s.gsub("^", "\\^").gsub("-", "")
+        job_name.tr(chars, "-")
+      end
+
+      # Illegal chars that should not be used in a job name
+      # A dash is assumed to be legal in job names in all batch schedulers
+      #
+      # @return [String] string of chars
+      def job_name_illegal_chars
+        ENV["OOD_JOB_NAME_ILLEGAL_CHARS"].to_s
+      end
     end
   end
 end

data/lib/ood_core/job/adapters/drmaa.rb

@@ -13,7 +13,7 @@
 #  The contents of this file are subject to the Sun Industry Standards
 #  Source License Version 1.2 (the "License"); You may not use this file
 #  except in compliance with the License. You may obtain a copy of the
-#  License at http://gridengine.sunsource.net/Gridengine_SISSL_license.html
+#  License at http://gridscheduler.sourceforge.net/Gridengine_SISSL_license.html
 #
 #  Software provided under this License is provided on an "AS IS" basis,
 #  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,

data/lib/ood_core/job/adapters/helper.rb

@@ -12,7 +12,26 @@ module OodCore
         def self.bin_path(cmd, bin_default, bin_overrides)
           bin_overrides.fetch(cmd.to_s) { Pathname.new(bin_default.to_s).join(cmd.to_s).to_s }
         end
+        
+        # Gets a command that submits command on another host via ssh
+        # @param submit_host [String] where to submit the command
+        # @param cmd [String] the desired command to execute on another host
+        # @param cmd_args [Array] arguments to the command specified above
+        # @param strict_host_checking [Bool] whether to use strict_host_checking
+        # @param env [Hash] env variables to be set w/ssh
+        #
+        # @return cmd [String] command wrapped in ssh if submit_host is present
+        # @return args [Array] command arguments including ssh_flags and original command
+        def self.ssh_wrap(submit_host, cmd, cmd_args, strict_host_checking = true, env = {})
+          return cmd, cmd_args if submit_host.to_s.empty?
+
+          check_host = strict_host_checking ? "yes" : "no"
+          args = ['-o', 'BatchMode=yes', '-o', 'UserKnownHostsFile=/dev/null', '-o', "StrictHostKeyChecking=#{check_host}", "#{submit_host}"]
+          env.each{|key, value| args.push("export #{key}=#{value};")}
+
+          return 'ssh', args + [cmd] + cmd_args
+        end
       end
     end
   end
-end
+end

data/lib/ood_core/job/adapters/linux_host.rb

@@ -106,7 +106,7 @@ module OodCore
         # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
-        def info_where_owner(owner: nil, attrs: nil)
+        def info_where_owner(_, attrs: nil)
           info_all
         end
 
@@ -201,6 +201,10 @@ module OodCore
           raise JobAdapterError, e.message
         end
 
+        def directive_prefix
+          nil
+        end
+
         private
 
         def host_permitted?(destination_host)

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -11,7 +11,7 @@ require 'time'
 class OodCore::Job::Adapters::LinuxHost::Launcher
   attr_reader :contain, :debug, :site_timeout, :session_name_label, :singularity_bin,
     :site_singularity_bindpath, :default_singularity_image, :ssh_hosts,
-    :strict_host_checking, :submit_host, :tmux_bin, :username
+    :strict_host_checking, :tmux_bin, :username
   # The root exception class that all LinuxHost adapter-specific exceptions inherit
   # from
   class Error < StandardError; end
@@ -57,7 +57,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   # @param hostname [#to_s] The hostname to submit the work to
   # @param script [OodCore::Job::Script] The script object defining the work
   def start_remote_session(script)
-    cmd = ssh_cmd(submit_host)
+    cmd = ssh_cmd(submit_host(script), ['/usr/bin/env', 'bash'])
 
     session_name = unique_session_name
     output = call(*cmd, stdin: wrapped_script(script, session_name))
@@ -67,13 +67,13 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   end
 
   def stop_remote_session(session_name, hostname)
-    cmd = ssh_cmd(hostname)
+    cmd = ssh_cmd(hostname, ['/usr/bin/env', 'bash'])
 
     kill_cmd = <<~SCRIPT
     # Get the tmux pane PID for the target session
     pane_pid=$(tmux list-panes -aF '\#{session_name} \#{pane_pid}' | grep '#{session_name}' | cut -f 2 -d ' ')
     # Find the Singularity sinit PID child of the pane process
-    pane_sinit_pid=$(pstree -p "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
+    pane_sinit_pid=$(pstree -p -l "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
     # Kill sinit which stops both Singularity-based processes and the tmux session
     kill "$pane_sinit_pid"
     SCRIPT
@@ -98,6 +98,14 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     }
   end
 
+  def submit_host(script = nil)
+    if script && script.native && script.native['submit_host_override']
+      script.native['submit_host_override']
+    else
+      @submit_host
+    end
+  end
+
   private
 
   # Call a forked Slurm command for a given cluster
@@ -108,19 +116,23 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     s.success? ? o : raise(Error, e)
   end
 
-  # The SSH invocation to send a command
+  # The full command to ssh into the destination host and execute the command.
+  # SSH options include:
   # -t Force pseudo-terminal allocation (required to allow tmux to run)
   # -o BatchMode=yes (set mode to be non-interactive)
   # if ! strict_host_checking
   # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
   # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
-  def ssh_cmd(destination_host)
+  #
+  # @param destination_host [#to_s] the destination host you wish to ssh into
+  # @param cmd [Array<#to_s>] the command to be executed on the destination host
+  def ssh_cmd(destination_host, cmd)
     if strict_host_checking
       [
         'ssh', '-t',
         '-o', 'BatchMode=yes',
         "#{username}@#{destination_host}"
-      ]
+      ].concat(cmd)
     else
       [
         'ssh', '-t',
@@ -128,7 +140,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         '-o', 'UserKnownHostsFile=/dev/null',
         '-o', 'StrictHostKeyChecking=no',
         "#{username}@#{destination_host}"
-      ]
+      ].concat(cmd)
     end
   end
 
@@ -162,6 +174,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'session_name' => session_name,
         'singularity_bin' => singularity_bin,
         'singularity_image' => singularity_image(script.native),
+        'ssh_hosts' => ssh_hosts,
         'tmux_bin' => tmux_bin,
       }.each{
         |key, value| bnd.local_variable_set(key, value)
@@ -237,7 +250,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
       ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
     )
     keys = [:session_name, :session_created, :session_pid]
-    cmd = ssh_cmd(destination_host) + ['tmux', 'list-panes', '-aF', format_str]
+    cmd = ssh_cmd(destination_host, ['tmux', 'list-panes', '-aF', format_str])
 
     call(*cmd).split(
       "\n"