ood_core 0.11.2 → 0.14.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,5 +1,6 @@
1
1
  require "ood_core/refinements/hash_extensions"
2
2
  require "ood_core/job/adapters/helper"
3
+ require 'shellwords'
3
4
 
4
5
  module OodCore
5
6
  module Job
@@ -9,16 +10,18 @@ module OodCore
9
10
  # Build the Torque adapter from a configuration
10
11
  # @param config [#to_h] the configuration for job adapter
11
12
  # @option config [#to_s] :host The batch server host
13
+ # @option config [#to_s] :submit_host The login node to submit the job via ssh
12
14
  # @option config [#to_s] :lib ('') Path to torque client libraries
13
15
  # @option config [#to_s] :bin ('') Path to torque client binaries
14
16
  # @option config [#to_h] :custom_bin ({}) Optional overrides to Torque client executables
15
17
  def self.build_torque(config)
16
18
  c = config.to_h.symbolize_keys
17
19
  host = c.fetch(:host) { raise ArgumentError, "No host specified. Missing argument: host" }.to_s
20
+ submit_host = c.fetch(:submit_host, "").to_s
18
21
  lib = c.fetch(:lib, "").to_s
19
22
  bin = c.fetch(:bin, "").to_s
20
23
  custom_bin = c.fetch(:custom_bin, {})
21
- pbs = Adapters::Torque::Batch.new(host: host, lib: lib, bin: bin, custom_bin: custom_bin)
24
+ pbs = Adapters::Torque::Batch.new(host: host, submit_host: submit_host, lib: lib, bin: bin, custom_bin: custom_bin)
22
25
  Adapters::Torque.new(pbs: pbs)
23
26
  end
24
27
  end
@@ -85,7 +88,7 @@ module OodCore
85
88
  depend << "afterany:#{afterany.join(':')}" unless afterany.empty?
86
89
 
87
90
  # Set mailing options
88
- mail_points = ""
91
+ mail_points = ""
89
92
  mail_points += "b" if script.email_on_started
90
93
  mail_points += "e" if script.email_on_terminated
91
94
 
@@ -129,40 +132,44 @@ module OodCore
129
132
  envvars.merge! script.native.fetch(:envvars, {})
130
133
  end
131
134
 
135
+ # Destructively change envvars to shellescape values
136
+ envvars.transform_values! { |v| Shellwords.escape(v) }
137
+
132
138
  # Submit job
133
139
  @pbs.submit_string(script.content, queue: script.queue_name, headers: headers, resources: resources, envvars: envvars)
134
140
  else
135
141
  # Set qsub arguments
136
142
  args = []
137
- args += ["-F", script.args.join(" ")] unless script.args.nil?
138
- args += ["-h"] if script.submit_as_hold
139
- args += ["-r", script.rerunnable ? "y" : "n"] unless script.rerunnable.nil?
140
- args += ["-M", script.email.join(",")] unless script.email.nil?
141
- args += ["-m", mail_points] unless mail_points.empty?
142
- args += ["-N", script.job_name] unless script.job_name.nil?
143
- args += ["-S", script.shell_path] unless script.shell_path.nil?
143
+ args.concat ["-F", script.args.join(" ")] unless script.args.nil?
144
+ args.concat ["-h"] if script.submit_as_hold
145
+ args.concat ["-r", script.rerunnable ? "y" : "n"] unless script.rerunnable.nil?
146
+ args.concat ["-M", script.email.join(",")] unless script.email.nil?
147
+ args.concat ["-m", mail_points] unless mail_points.empty?
148
+ args.concat ["-N", script.job_name] unless script.job_name.nil?
149
+ args.concat ["-S", script.shell_path] unless script.shell_path.nil?
144
150
  # ignore input_path (not defined in Torque)
145
- args += ["-o", script.output_path] unless script.output_path.nil?
146
- args += ["-e", script.error_path] unless script.error_path.nil?
147
- args += ["-W", "x=advres:#{script.reservation_id}"] unless script.reservation_id.nil?
148
- args += ["-q", script.queue_name] unless script.queue_name.nil?
149
- args += ["-p", script.priority] unless script.priority.nil?
150
- args += ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
151
- args += ["-A", script.accounting_id] unless script.accounting_id.nil?
152
- args += ["-W", "depend=#{depend.join(",")}"] unless depend.empty?
153
- args += ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
154
- args += ['-t', script.job_array_request] unless script.job_array_request.nil?
151
+ args.concat ["-o", script.output_path] unless script.output_path.nil?
152
+ args.concat ["-e", script.error_path] unless script.error_path.nil?
153
+ args.concat ["-W", "x=advres:#{script.reservation_id}"] unless script.reservation_id.nil?
154
+ args.concat ["-q", script.queue_name] unless script.queue_name.nil?
155
+ args.concat ["-p", script.priority] unless script.priority.nil?
156
+ args.concat ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
157
+ args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
158
+ args.concat ["-W", "depend=#{depend.join(",")}"] unless depend.empty?
159
+ args.concat ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
160
+ args.concat ['-t', script.job_array_request] unless script.job_array_request.nil?
161
+ args.concat ['-l', "qos=#{script.qos}"] unless script.qos.nil?
155
162
  # Set environment variables
156
163
  env = script.job_environment.to_h
157
- args += ["-v", env.keys.join(",")] unless env.empty?
158
- args += ["-V"] if script.copy_environment?
164
+ args.concat ["-v", env.keys.join(",")] unless env.empty?
165
+ args.concat ["-V"] if script.copy_environment?
159
166
 
160
167
  # If error_path is not specified we join stdout & stderr (as this
161
168
  # mimics what the other resource managers do)
162
- args += ["-j", "oe"] if script.error_path.nil?
169
+ args.concat ["-j", "oe"] if script.error_path.nil?
163
170
 
164
171
  # Set native options
165
- args += script.native if script.native
172
+ args.concat script.native if script.native
166
173
 
167
174
  # Submit job
168
175
  @pbs.submit(script.content, args: args, env: env, chdir: script.workdir)
@@ -9,6 +9,18 @@ class OodCore::Job::Adapters::Torque
9
9
  # @return [String] the batch server host
10
10
  attr_reader :host
11
11
 
12
+ # The login node where job is submitted via ssh
13
+ # @example OSC's owens login node
14
+ # my_conn.submit_host #=> "owens.osc.edu"
15
+ # @return [String] the login node
16
+ attr_reader :submit_host
17
+
18
+ # Determines whether to use strict_host_checking for ssh
19
+ # @example
20
+ # my_conn.strict_host_checking.to_s #=> "owens.osc.edu"
21
+ # @return [Bool]
22
+ attr_reader :strict_host_checking
23
+
12
24
  # The path to the Torque client installation libraries
13
25
  # @example For Torque 5.0.0
14
26
  # my_conn.lib.to_s #=> "/usr/local/Torque/5.0.0/lib"
@@ -32,19 +44,23 @@ class OodCore::Job::Adapters::Torque
32
44
  class Error < StandardError; end
33
45
 
34
46
  # @param host [#to_s] the batch server host
47
+ # @param submit_host [#to_s] the login node
48
+ # @param strict_host_checking [bool] use strict host checking when ssh to submit_host
35
49
  # @param lib [#to_s] path to FFI installation libraries
36
50
  # @param bin [#to_s] path to FFI installation binaries
37
- def initialize(host:, lib: "", bin: "", bin_overrides: {}, **_)
38
- @host = host.to_s
39
- @lib = Pathname.new(lib.to_s)
40
- @bin = Pathname.new(bin.to_s)
41
- @bin_overrides = bin_overrides
51
+ def initialize(host:, submit_host: "", strict_host_checking: true, lib: "", bin: "", bin_overrides: {}, **_)
52
+ @host = host.to_s
53
+ @submit_host = submit_host.to_s
54
+ @strict_host_checking = strict_host_checking
55
+ @lib = Pathname.new(lib.to_s)
56
+ @bin = Pathname.new(bin.to_s)
57
+ @bin_overrides = bin_overrides
42
58
  end
43
59
 
44
60
  # Convert object to hash
45
61
  # @return [Hash] the hash describing this object
46
62
  def to_h
47
- {host: host, lib: lib, bin: bin}
63
+ {host: host, submit_host: submit_host, strict_host_checking: strict_host_checking, lib: lib, bin: bin}
48
64
  end
49
65
 
50
66
  # The comparison operator
@@ -437,10 +453,10 @@ class OodCore::Job::Adapters::Torque
437
453
  # NB: The binary includes many useful filters and is preferred
438
454
  def qsub_submit(script, queue, headers, resources, envvars)
439
455
  params = []
440
- params += ["-q", "#{queue}"] unless queue.empty?
441
- params += headers.map {|k,v| qsub_arg(k,v)}.flatten
442
- params += resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten
443
- params += ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
456
+ params.concat ["-q", "#{queue}"] unless queue.empty?
457
+ params.concat headers.map {|k,v| qsub_arg(k,v)}.flatten
458
+ params.concat resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten
459
+ params.concat ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
444
460
  params << script
445
461
 
446
462
  env = {
@@ -448,6 +464,7 @@ class OodCore::Job::Adapters::Torque
448
464
  "LD_LIBRARY_PATH" => "#{lib}:#{ENV['LD_LIBRARY_PATH']}"
449
465
  }
450
466
  cmd = OodCore::Job::Adapters::Helper.bin_path('qsub', bin, bin_overrides)
467
+ cmd, params = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, params, strict_host_checking, env)
451
468
  o, e, s = Open3.capture3(env, cmd, *params)
452
469
  raise Error, e unless s.success?
453
470
  o.chomp
@@ -456,14 +473,14 @@ class OodCore::Job::Adapters::Torque
456
473
  # Call a forked PBS command for a given host
457
474
  def call(cmd, *args, env: {}, stdin: "", chdir: nil)
458
475
  cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
459
- args = args.map(&:to_s)
460
476
  env = env.to_h.each_with_object({}) {|(k,v), h| h[k.to_s] = v.to_s}.merge({
461
477
  "PBS_DEFAULT" => host,
462
478
  "LD_LIBRARY_PATH" => %{#{lib}:#{ENV["LD_LIBRARY_PATH"]}}
463
479
  })
480
+ cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking, env)
464
481
  stdin = stdin.to_s
465
482
  chdir ||= "."
466
- o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin, chdir: chdir.to_s)
483
+ o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin, chdir: chdir.to_s)
467
484
  s.success? ? o : raise(Error, e)
468
485
  end
469
486
  end
@@ -99,6 +99,10 @@ module OodCore
99
99
  # @return [String, nil] job array request
100
100
  attr_reader :job_array_request
101
101
 
102
+ # The qos selected for the job
103
+ # @return [String, nil] qos
104
+ attr_reader :qos
105
+
102
106
  # Object detailing any native specifications that are implementation specific
103
107
  # @note Should not be used at all costs.
104
108
  # @return [Object, nil] native specifications
@@ -130,7 +134,10 @@ module OodCore
130
134
  # @param start_time [#to_i, nil] eligible start time
131
135
  # @param wall_time [#to_i, nil] max real time
132
136
  # @param accounting_id [#to_s, nil] accounting id
137
+ # @param job_array_request [#to_s, nil] job array request
138
+ # @param qos [#to_s, nil] qos
133
139
  # @param native [Object, nil] native specifications
140
+ # @param copy_environment [Boolean, nil] copy the environment
134
141
  def initialize(content:, args: nil, submit_as_hold: nil, rerunnable: nil,
135
142
  job_environment: nil, workdir: nil, email: nil,
136
143
  email_on_started: nil, email_on_terminated: nil,
@@ -138,7 +145,7 @@ module OodCore
138
145
  output_path: nil, error_path: nil, reservation_id: nil,
139
146
  queue_name: nil, priority: nil, start_time: nil,
140
147
  wall_time: nil, accounting_id: nil, job_array_request: nil,
141
- native: nil, copy_environment: nil, **_)
148
+ qos: nil, native: nil, copy_environment: nil, **_)
142
149
  @content = content.to_s
143
150
 
144
151
  @submit_as_hold = submit_as_hold
@@ -162,6 +169,7 @@ module OodCore
162
169
  @wall_time = wall_time && wall_time.to_i
163
170
  @accounting_id = accounting_id && accounting_id.to_s
164
171
  @job_array_request = job_array_request && job_array_request.to_s
172
+ @qos = qos && qos.to_s
165
173
  @native = native
166
174
  @copy_environment = (copy_environment.nil?) ? nil : !! copy_environment
167
175
  end
@@ -191,6 +199,7 @@ module OodCore
191
199
  wall_time: wall_time,
192
200
  accounting_id: accounting_id,
193
201
  job_array_request: job_array_request,
202
+ qos: qos,
194
203
  native: native,
195
204
  copy_environment: copy_environment
196
205
  }
@@ -1,4 +1,4 @@
1
1
  module OodCore
2
2
  # The current version of {OodCore}
3
- VERSION = "0.11.2"
3
+ VERSION = "0.14.0"
4
4
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ood_core
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.11.2
4
+ version: 0.14.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Franz
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2020-04-24 00:00:00.000000000 Z
13
+ date: 2020-10-01 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: ood_support
@@ -163,8 +163,14 @@ files:
163
163
  - lib/ood_core/errors.rb
164
164
  - lib/ood_core/invalid_cluster.rb
165
165
  - lib/ood_core/job/adapter.rb
166
+ - lib/ood_core/job/adapters/ccq.rb
166
167
  - lib/ood_core/job/adapters/drmaa.rb
167
168
  - lib/ood_core/job/adapters/helper.rb
169
+ - lib/ood_core/job/adapters/kubernetes.rb
170
+ - lib/ood_core/job/adapters/kubernetes/batch.rb
171
+ - lib/ood_core/job/adapters/kubernetes/helper.rb
172
+ - lib/ood_core/job/adapters/kubernetes/resources.rb
173
+ - lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb
168
174
  - lib/ood_core/job/adapters/linux_host.rb
169
175
  - lib/ood_core/job/adapters/linux_host/launcher.rb
170
176
  - lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
@@ -215,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
215
221
  - !ruby/object:Gem::Version
216
222
  version: '0'
217
223
  requirements: []
218
- rubygems_version: 3.0.3
224
+ rubygems_version: 3.0.8
219
225
  signing_key:
220
226
  specification_version: 4
221
227
  summary: Open OnDemand core library