ood_core 0.11.3 → 0.15.0

data/lib/ood_core/job/adapters/torque.rb

@@ -1,5 +1,6 @@
 require "ood_core/refinements/hash_extensions"
 require "ood_core/job/adapters/helper"
+require 'shellwords'
 
 module OodCore
   module Job
@@ -9,16 +10,18 @@ module OodCore
       # Build the Torque adapter from a configuration
       # @param config [#to_h] the configuration for job adapter
       # @option config [#to_s] :host The batch server host
+      # @option config [#to_s] :submit_host The login node to submit the job via ssh
       # @option config [#to_s] :lib ('') Path to torque client libraries
       # @option config [#to_s] :bin ('') Path to torque client binaries
       # @option config [#to_h] :custom_bin ({}) Optional overrides to Torque client executables
       def self.build_torque(config)
         c = config.to_h.symbolize_keys
         host = c.fetch(:host) { raise ArgumentError, "No host specified. Missing argument: host" }.to_s
+        submit_host = c.fetch(:submit_host, "").to_s
         lib  = c.fetch(:lib, "").to_s
         bin  = c.fetch(:bin, "").to_s
         custom_bin = c.fetch(:custom_bin, {})
-        pbs  = Adapters::Torque::Batch.new(host: host, lib: lib, bin: bin, custom_bin: custom_bin)
+        pbs  = Adapters::Torque::Batch.new(host: host, submit_host: submit_host, lib: lib, bin: bin, custom_bin: custom_bin)
         Adapters::Torque.new(pbs: pbs)
       end
     end
@@ -85,7 +88,7 @@ module OodCore
           depend << "afterany:#{afterany.join(':')}"     unless afterany.empty?
 
           # Set mailing options
-          mail_points  = ""
+          mail_points = ""
           mail_points += "b" if script.email_on_started
           mail_points += "e" if script.email_on_terminated
 
@@ -129,40 +132,44 @@ module OodCore
               envvars.merge!   script.native.fetch(:envvars, {})
             end
 
+            # Destructively change envvars to shellescape values
+            envvars.transform_values! { |v| Shellwords.escape(v) }
+
             # Submit job
             @pbs.submit_string(script.content, queue: script.queue_name, headers: headers, resources: resources, envvars: envvars)
           else
             # Set qsub arguments
             args = []
-            args += ["-F", script.args.join(" ")] unless script.args.nil?
-            args += ["-h"] if script.submit_as_hold
-            args += ["-r", script.rerunnable ? "y" : "n"] unless script.rerunnable.nil?
-            args += ["-M", script.email.join(",")] unless script.email.nil?
-            args += ["-m", mail_points] unless mail_points.empty?
-            args += ["-N", script.job_name] unless script.job_name.nil?
-            args += ["-S", script.shell_path] unless script.shell_path.nil?
+            args.concat ["-F", script.args.join(" ")] unless script.args.nil?
+            args.concat ["-h"] if script.submit_as_hold
+            args.concat ["-r", script.rerunnable ? "y" : "n"] unless script.rerunnable.nil?
+            args.concat ["-M", script.email.join(",")] unless script.email.nil?
+            args.concat ["-m", mail_points] unless mail_points.empty?
+            args.concat ["-N", script.job_name] unless script.job_name.nil?
+            args.concat ["-S", script.shell_path] unless script.shell_path.nil?
             # ignore input_path (not defined in Torque)
-            args += ["-o", script.output_path] unless script.output_path.nil?
-            args += ["-e", script.error_path] unless script.error_path.nil?
-            args += ["-W", "x=advres:#{script.reservation_id}"] unless script.reservation_id.nil?
-            args += ["-q", script.queue_name] unless script.queue_name.nil?
-            args += ["-p", script.priority] unless script.priority.nil?
-            args += ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
-            args += ["-A", script.accounting_id] unless script.accounting_id.nil?
-            args += ["-W", "depend=#{depend.join(",")}"] unless depend.empty?
-            args += ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
-            args += ['-t', script.job_array_request] unless script.job_array_request.nil?
+            args.concat ["-o", script.output_path] unless script.output_path.nil?
+            args.concat ["-e", script.error_path] unless script.error_path.nil?
+            args.concat ["-W", "x=advres:#{script.reservation_id}"] unless script.reservation_id.nil?
+            args.concat ["-q", script.queue_name] unless script.queue_name.nil?
+            args.concat ["-p", script.priority] unless script.priority.nil?
+            args.concat ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
+            args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
+            args.concat ["-W", "depend=#{depend.join(",")}"] unless depend.empty?
+            args.concat ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
+            args.concat ['-t', script.job_array_request] unless script.job_array_request.nil?
+            args.concat ['-l', "qos=#{script.qos}"] unless script.qos.nil?
             # Set environment variables
             env = script.job_environment.to_h
-            args += ["-v", env.keys.join(",")] unless env.empty?
-            args += ["-V"] if script.copy_environment?
+            args.concat ["-v", env.keys.join(",")] unless env.empty?
+            args.concat ["-V"] if script.copy_environment?
 
             # If error_path is not specified we join stdout & stderr (as this
             # mimics what the other resource managers do)
-            args += ["-j", "oe"] if script.error_path.nil?
+            args.concat ["-j", "oe"] if script.error_path.nil?
 
             # Set native options
-            args += script.native if script.native
+            args.concat script.native if script.native
 
             # Submit job
             @pbs.submit(script.content, args: args, env: env, chdir: script.workdir)

data/lib/ood_core/job/adapters/torque/batch.rb

@@ -9,6 +9,18 @@ class OodCore::Job::Adapters::Torque
     # @return [String] the batch server host
     attr_reader :host
 
+    # The login node where job is submitted via ssh
+    # @example OSC's owens login node
+    #   my_conn.submit_host #=> "owens.osc.edu"
+    # @return [String] the login node
+    attr_reader :submit_host
+
+    # Determines whether to use strict_host_checking for ssh
+    # @example 
+    #   my_conn.strict_host_checking.to_s #=> "owens.osc.edu"
+    # @return [Bool] 
+    attr_reader :strict_host_checking
+
     # The path to the Torque client installation libraries
     # @example For Torque 5.0.0
     #   my_conn.lib.to_s #=> "/usr/local/Torque/5.0.0/lib"
@@ -32,19 +44,23 @@ class OodCore::Job::Adapters::Torque
     class Error < StandardError; end
 
     # @param host [#to_s] the batch server host
+    # @param submit_host [#to_s] the login node
+    # @param strict_host_checking [bool] use strict host checking when ssh to submit_host
     # @param lib [#to_s] path to FFI installation libraries
     # @param bin [#to_s] path to FFI installation binaries
-    def initialize(host:, lib: "", bin: "", bin_overrides: {}, **_)
-      @host    = host.to_s
-      @lib     = Pathname.new(lib.to_s)
-      @bin     = Pathname.new(bin.to_s)
-      @bin_overrides = bin_overrides
+    def initialize(host:, submit_host: "", strict_host_checking: true, lib: "", bin: "", bin_overrides: {}, **_)
+      @host                 = host.to_s
+      @submit_host          = submit_host.to_s
+      @strict_host_checking = strict_host_checking
+      @lib                  = Pathname.new(lib.to_s)
+      @bin                  = Pathname.new(bin.to_s)
+      @bin_overrides        = bin_overrides
     end
 
     # Convert object to hash
     # @return [Hash] the hash describing this object
     def to_h
-      {host: host, lib: lib, bin: bin}
+      {host: host, submit_host: submit_host, strict_host_checking: strict_host_checking, lib: lib, bin: bin}
     end
 
     # The comparison operator
@@ -437,10 +453,10 @@ class OodCore::Job::Adapters::Torque
       # NB: The binary includes many useful filters and is preferred
       def qsub_submit(script, queue, headers, resources, envvars)
         params  = []
-        params += ["-q", "#{queue}"] unless queue.empty?
-        params += headers.map {|k,v| qsub_arg(k,v)}.flatten
-        params += resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten
-        params += ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
+        params.concat ["-q", "#{queue}"] unless queue.empty?
+        params.concat headers.map {|k,v| qsub_arg(k,v)}.flatten
+        params.concat resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten
+        params.concat ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
         params << script
 
         env = {
@@ -448,6 +464,7 @@ class OodCore::Job::Adapters::Torque
           "LD_LIBRARY_PATH" => "#{lib}:#{ENV['LD_LIBRARY_PATH']}"
         }
         cmd = OodCore::Job::Adapters::Helper.bin_path('qsub', bin, bin_overrides)
+        cmd, params = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, params, strict_host_checking, env)
         o, e, s = Open3.capture3(env, cmd, *params)
         raise Error, e unless s.success?
         o.chomp
@@ -456,14 +473,14 @@ class OodCore::Job::Adapters::Torque
       # Call a forked PBS command for a given host
       def call(cmd, *args, env: {}, stdin: "", chdir: nil)
         cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
-        args = args.map(&:to_s)
         env  = env.to_h.each_with_object({}) {|(k,v), h| h[k.to_s] = v.to_s}.merge({
           "PBS_DEFAULT"     => host,
           "LD_LIBRARY_PATH" => %{#{lib}:#{ENV["LD_LIBRARY_PATH"]}}
         })
+        cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking, env)
         stdin = stdin.to_s
         chdir ||= "."
-        o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin, chdir: chdir.to_s)
+        o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin, chdir: chdir.to_s)
         s.success? ? o : raise(Error, e)
       end
   end

data/lib/ood_core/job/script.rb

@@ -99,6 +99,10 @@ module OodCore
       # @return [String, nil] job array request
       attr_reader :job_array_request
 
+      # The qos selected for the job
+      # @return [String, nil] qos
+      attr_reader :qos
+
       # Object detailing any native specifications that are implementation specific
       # @note Should not be used at all costs.
       # @return [Object, nil] native specifications
@@ -130,7 +134,10 @@ module OodCore
       # @param start_time [#to_i, nil] eligible start time
       # @param wall_time [#to_i, nil] max real time
       # @param accounting_id [#to_s, nil] accounting id
+      # @param job_array_request [#to_s, nil] job array request
+      # @param qos [#to_s, nil] qos
       # @param native [Object, nil] native specifications
+      # @param copy_environment [Boolean, nil] copy the environment
       def initialize(content:, args: nil, submit_as_hold: nil, rerunnable: nil,
                      job_environment: nil, workdir: nil, email: nil,
                      email_on_started: nil, email_on_terminated: nil,
@@ -138,7 +145,7 @@ module OodCore
                      output_path: nil, error_path: nil, reservation_id: nil,
                      queue_name: nil, priority: nil, start_time: nil,
                      wall_time: nil, accounting_id: nil, job_array_request: nil,
-                     native: nil, copy_environment: nil, **_)
+                     qos: nil, native: nil, copy_environment: nil, **_)
         @content = content.to_s
 
         @submit_as_hold      = submit_as_hold
@@ -162,6 +169,7 @@ module OodCore
         @wall_time          = wall_time         && wall_time.to_i
         @accounting_id      = accounting_id     && accounting_id.to_s
         @job_array_request  = job_array_request && job_array_request.to_s
+        @qos                = qos               && qos.to_s
         @native             = native
         @copy_environment   = (copy_environment.nil?) ? nil : !! copy_environment
       end
@@ -191,6 +199,7 @@ module OodCore
           wall_time:           wall_time,
           accounting_id:       accounting_id,
           job_array_request:   job_array_request,
+          qos:                 qos,
           native:              native,
           copy_environment:    copy_environment
         }

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.11.3"
+  VERSION = "0.15.0"
 end

data/ood_core.gemspec

@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
   spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_runtime_dependency "activesupport", ">= 5.2", "< 6.0"
   spec.add_development_dependency "rake", "~> 13.0.1"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.15.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-11 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -52,14 +52,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -140,9 +160,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -163,8 +183,15 @@ files:
 - lib/ood_core/errors.rb
 - lib/ood_core/invalid_cluster.rb
 - lib/ood_core/job/adapter.rb
+- lib/ood_core/job/adapters/ccq.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
+- lib/ood_core/job/adapters/kubernetes.rb
+- lib/ood_core/job/adapters/kubernetes/batch.rb
+- lib/ood_core/job/adapters/kubernetes/helper.rb
+- lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb
+- lib/ood_core/job/adapters/kubernetes/resources.rb
+- lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb
 - lib/ood_core/job/adapters/linux_host.rb
 - lib/ood_core/job/adapters/linux_host/launcher.rb
 - lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
@@ -215,8 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Open OnDemand core library

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5.5
-before_install: gem install bundler -v '~> 1.17'
-notifications:
-  email:
-    on_success: never
-    on_failure: always