ood_core 0.11.4 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +30 -0
- data/CHANGELOG.md +55 -1
- data/README.md +7 -6
- data/lib/ood_core/job/adapters/ccq.rb +274 -0
- data/lib/ood_core/job/adapters/helper.rb +20 -1
- data/lib/ood_core/job/adapters/kubernetes.rb +193 -0
- data/lib/ood_core/job/adapters/kubernetes/batch.rb +354 -0
- data/lib/ood_core/job/adapters/kubernetes/helper.rb +294 -0
- data/lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb +9 -0
- data/lib/ood_core/job/adapters/kubernetes/resources.rb +58 -0
- data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb +158 -0
- data/lib/ood_core/job/adapters/linux_host/launcher.rb +10 -1
- data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh +18 -15
- data/lib/ood_core/job/adapters/lsf.rb +1 -0
- data/lib/ood_core/job/adapters/lsf/batch.rb +5 -3
- data/lib/ood_core/job/adapters/lsf/helper.rb +22 -22
- data/lib/ood_core/job/adapters/pbspro.rb +54 -34
- data/lib/ood_core/job/adapters/sge/batch.rb +6 -5
- data/lib/ood_core/job/adapters/sge/helper.rb +19 -19
- data/lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb +35 -4
- data/lib/ood_core/job/adapters/sge/qstat_xml_r_listener.rb +25 -2
- data/lib/ood_core/job/adapters/slurm.rb +79 -38
- data/lib/ood_core/job/adapters/torque.rb +30 -23
- data/lib/ood_core/job/adapters/torque/batch.rb +29 -12
- data/lib/ood_core/job/script.rb +9 -1
- data/lib/ood_core/version.rb +1 -1
- data/ood_core.gemspec +2 -1
- metadata +33 -6
- data/.travis.yml +0 -9
@@ -9,6 +9,18 @@ class OodCore::Job::Adapters::Torque
|
|
9
9
|
# @return [String] the batch server host
|
10
10
|
attr_reader :host
|
11
11
|
|
12
|
+
# The login node where job is submitted via ssh
|
13
|
+
# @example OSC's owens login node
|
14
|
+
# my_conn.submit_host #=> "owens.osc.edu"
|
15
|
+
# @return [String] the login node
|
16
|
+
attr_reader :submit_host
|
17
|
+
|
18
|
+
# Determines whether to use strict_host_checking for ssh
|
19
|
+
# @example
|
20
|
+
# my_conn.strict_host_checking.to_s #=> "owens.osc.edu"
|
21
|
+
# @return [Bool]
|
22
|
+
attr_reader :strict_host_checking
|
23
|
+
|
12
24
|
# The path to the Torque client installation libraries
|
13
25
|
# @example For Torque 5.0.0
|
14
26
|
# my_conn.lib.to_s #=> "/usr/local/Torque/5.0.0/lib"
|
@@ -32,19 +44,23 @@ class OodCore::Job::Adapters::Torque
|
|
32
44
|
class Error < StandardError; end
|
33
45
|
|
34
46
|
# @param host [#to_s] the batch server host
|
47
|
+
# @param submit_host [#to_s] the login node
|
48
|
+
# @param strict_host_checking [bool] use strict host checking when ssh to submit_host
|
35
49
|
# @param lib [#to_s] path to FFI installation libraries
|
36
50
|
# @param bin [#to_s] path to FFI installation binaries
|
37
|
-
def initialize(host:, lib: "", bin: "", bin_overrides: {}, **_)
|
38
|
-
@host
|
39
|
-
@
|
40
|
-
@
|
41
|
-
@
|
51
|
+
def initialize(host:, submit_host: "", strict_host_checking: true, lib: "", bin: "", bin_overrides: {}, **_)
|
52
|
+
@host = host.to_s
|
53
|
+
@submit_host = submit_host.to_s
|
54
|
+
@strict_host_checking = strict_host_checking
|
55
|
+
@lib = Pathname.new(lib.to_s)
|
56
|
+
@bin = Pathname.new(bin.to_s)
|
57
|
+
@bin_overrides = bin_overrides
|
42
58
|
end
|
43
59
|
|
44
60
|
# Convert object to hash
|
45
61
|
# @return [Hash] the hash describing this object
|
46
62
|
def to_h
|
47
|
-
{host: host, lib: lib, bin: bin}
|
63
|
+
{host: host, submit_host: submit_host, strict_host_checking: strict_host_checking, lib: lib, bin: bin}
|
48
64
|
end
|
49
65
|
|
50
66
|
# The comparison operator
|
@@ -437,10 +453,10 @@ class OodCore::Job::Adapters::Torque
|
|
437
453
|
# NB: The binary includes many useful filters and is preferred
|
438
454
|
def qsub_submit(script, queue, headers, resources, envvars)
|
439
455
|
params = []
|
440
|
-
params
|
441
|
-
params
|
442
|
-
params
|
443
|
-
params
|
456
|
+
params.concat ["-q", "#{queue}"] unless queue.empty?
|
457
|
+
params.concat headers.map {|k,v| qsub_arg(k,v)}.flatten
|
458
|
+
params.concat resources.map{|k,v| ["-l", "#{k}=#{v}"]}.flatten
|
459
|
+
params.concat ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
|
444
460
|
params << script
|
445
461
|
|
446
462
|
env = {
|
@@ -448,6 +464,7 @@ class OodCore::Job::Adapters::Torque
|
|
448
464
|
"LD_LIBRARY_PATH" => "#{lib}:#{ENV['LD_LIBRARY_PATH']}"
|
449
465
|
}
|
450
466
|
cmd = OodCore::Job::Adapters::Helper.bin_path('qsub', bin, bin_overrides)
|
467
|
+
cmd, params = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, params, strict_host_checking, env)
|
451
468
|
o, e, s = Open3.capture3(env, cmd, *params)
|
452
469
|
raise Error, e unless s.success?
|
453
470
|
o.chomp
|
@@ -456,14 +473,14 @@ class OodCore::Job::Adapters::Torque
|
|
456
473
|
# Call a forked PBS command for a given host
|
457
474
|
def call(cmd, *args, env: {}, stdin: "", chdir: nil)
|
458
475
|
cmd = OodCore::Job::Adapters::Helper.bin_path(cmd, bin, bin_overrides)
|
459
|
-
args = args.map(&:to_s)
|
460
476
|
env = env.to_h.each_with_object({}) {|(k,v), h| h[k.to_s] = v.to_s}.merge({
|
461
477
|
"PBS_DEFAULT" => host,
|
462
478
|
"LD_LIBRARY_PATH" => %{#{lib}:#{ENV["LD_LIBRARY_PATH"]}}
|
463
479
|
})
|
480
|
+
cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking, env)
|
464
481
|
stdin = stdin.to_s
|
465
482
|
chdir ||= "."
|
466
|
-
o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin, chdir: chdir.to_s)
|
483
|
+
o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin, chdir: chdir.to_s)
|
467
484
|
s.success? ? o : raise(Error, e)
|
468
485
|
end
|
469
486
|
end
|
data/lib/ood_core/job/script.rb
CHANGED
@@ -99,6 +99,10 @@ module OodCore
|
|
99
99
|
# @return [String, nil] job array request
|
100
100
|
attr_reader :job_array_request
|
101
101
|
|
102
|
+
# The qos selected for the job
|
103
|
+
# @return [String, nil] qos
|
104
|
+
attr_reader :qos
|
105
|
+
|
102
106
|
# Object detailing any native specifications that are implementation specific
|
103
107
|
# @note Should not be used at all costs.
|
104
108
|
# @return [Object, nil] native specifications
|
@@ -130,6 +134,8 @@ module OodCore
|
|
130
134
|
# @param start_time [#to_i, nil] eligible start time
|
131
135
|
# @param wall_time [#to_i, nil] max real time
|
132
136
|
# @param accounting_id [#to_s, nil] accounting id
|
137
|
+
# @param job_array_request [#to_s, nil] job array request
|
138
|
+
# @param qos [#to_s, nil] qos
|
133
139
|
# @param native [Object, nil] native specifications
|
134
140
|
# @param copy_environment [Boolean, nil] copy the environment
|
135
141
|
def initialize(content:, args: nil, submit_as_hold: nil, rerunnable: nil,
|
@@ -139,7 +145,7 @@ module OodCore
|
|
139
145
|
output_path: nil, error_path: nil, reservation_id: nil,
|
140
146
|
queue_name: nil, priority: nil, start_time: nil,
|
141
147
|
wall_time: nil, accounting_id: nil, job_array_request: nil,
|
142
|
-
native: nil, copy_environment: nil, **_)
|
148
|
+
qos: nil, native: nil, copy_environment: nil, **_)
|
143
149
|
@content = content.to_s
|
144
150
|
|
145
151
|
@submit_as_hold = submit_as_hold
|
@@ -163,6 +169,7 @@ module OodCore
|
|
163
169
|
@wall_time = wall_time && wall_time.to_i
|
164
170
|
@accounting_id = accounting_id && accounting_id.to_s
|
165
171
|
@job_array_request = job_array_request && job_array_request.to_s
|
172
|
+
@qos = qos && qos.to_s
|
166
173
|
@native = native
|
167
174
|
@copy_environment = (copy_environment.nil?) ? nil : !! copy_environment
|
168
175
|
end
|
@@ -192,6 +199,7 @@ module OodCore
|
|
192
199
|
wall_time: wall_time,
|
193
200
|
accounting_id: accounting_id,
|
194
201
|
job_array_request: job_array_request,
|
202
|
+
qos: qos,
|
195
203
|
native: native,
|
196
204
|
copy_environment: copy_environment
|
197
205
|
}
|
data/lib/ood_core/version.rb
CHANGED
data/ood_core.gemspec
CHANGED
@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
|
|
24
24
|
|
25
25
|
spec.add_runtime_dependency "ood_support", "~> 0.0.2"
|
26
26
|
spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
|
27
|
-
spec.add_development_dependency "bundler", "~> 1
|
27
|
+
spec.add_development_dependency "bundler", "~> 2.1"
|
28
|
+
spec.add_runtime_dependency "activesupport", ">= 5.2", "< 6.0"
|
28
29
|
spec.add_development_dependency "rake", "~> 13.0.1"
|
29
30
|
spec.add_development_dependency "rspec", "~> 3.0"
|
30
31
|
spec.add_development_dependency "pry", "~> 0.10"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ood_core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.15.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Franz
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-02-25 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: ood_support
|
@@ -52,14 +52,34 @@ dependencies:
|
|
52
52
|
requirements:
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '1
|
55
|
+
version: '2.1'
|
56
56
|
type: :development
|
57
57
|
prerelease: false
|
58
58
|
version_requirements: !ruby/object:Gem::Requirement
|
59
59
|
requirements:
|
60
60
|
- - "~>"
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version: '1
|
62
|
+
version: '2.1'
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: activesupport
|
65
|
+
requirement: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '5.2'
|
70
|
+
- - "<"
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '6.0'
|
73
|
+
type: :runtime
|
74
|
+
prerelease: false
|
75
|
+
version_requirements: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '5.2'
|
80
|
+
- - "<"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '6.0'
|
63
83
|
- !ruby/object:Gem::Dependency
|
64
84
|
name: rake
|
65
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -140,9 +160,9 @@ executables: []
|
|
140
160
|
extensions: []
|
141
161
|
extra_rdoc_files: []
|
142
162
|
files:
|
163
|
+
- ".github/workflows/test.yml"
|
143
164
|
- ".gitignore"
|
144
165
|
- ".rspec"
|
145
|
-
- ".travis.yml"
|
146
166
|
- CHANGELOG.md
|
147
167
|
- Gemfile
|
148
168
|
- LICENSE.txt
|
@@ -163,8 +183,15 @@ files:
|
|
163
183
|
- lib/ood_core/errors.rb
|
164
184
|
- lib/ood_core/invalid_cluster.rb
|
165
185
|
- lib/ood_core/job/adapter.rb
|
186
|
+
- lib/ood_core/job/adapters/ccq.rb
|
166
187
|
- lib/ood_core/job/adapters/drmaa.rb
|
167
188
|
- lib/ood_core/job/adapters/helper.rb
|
189
|
+
- lib/ood_core/job/adapters/kubernetes.rb
|
190
|
+
- lib/ood_core/job/adapters/kubernetes/batch.rb
|
191
|
+
- lib/ood_core/job/adapters/kubernetes/helper.rb
|
192
|
+
- lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb
|
193
|
+
- lib/ood_core/job/adapters/kubernetes/resources.rb
|
194
|
+
- lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb
|
168
195
|
- lib/ood_core/job/adapters/linux_host.rb
|
169
196
|
- lib/ood_core/job/adapters/linux_host/launcher.rb
|
170
197
|
- lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
|
@@ -215,7 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
215
242
|
- !ruby/object:Gem::Version
|
216
243
|
version: '0'
|
217
244
|
requirements: []
|
218
|
-
rubygems_version: 3.
|
245
|
+
rubygems_version: 3.1.2
|
219
246
|
signing_key:
|
220
247
|
specification_version: 4
|
221
248
|
summary: Open OnDemand core library
|