ood_core 0.26.1 → 0.27.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3ef6cd5a8a206393dcaed9d0121e3646dd6fc3d2b0405992c3fcfa5a745e9489
4
- data.tar.gz: 16a5cf6f03ed2be4c563dafb7eaa9e2a04d465a4d8a3cea5d3bb14b634d65868
3
+ metadata.gz: c4cf552caaaedcb74de1091d8e9d1503b21f140b961ea8ccc3e53c1a25fa88f9
4
+ data.tar.gz: e69ec942c413e569d232ed8e88ee9a0abb79d5f3e9c7bf2d17cce7d3fe92b465
5
5
  SHA512:
6
- metadata.gz: 563930fc05d105b46fa1f8942294a06b3fd3ffce665ba6aa93fae3a21759613fdf229c9cca4f5ec8bd99c7b57cbb9e7e4348420de47d4d0884a78dfd62c43b5f
7
- data.tar.gz: dc88b7bc4a28e96725bcd2caf0e40659bca94a40ab1c50432af0eee2b649f4a48356e858a7f9ceca1a8a18f99c823bf9ffc4c7cecefecb9820d8ce7621c7c51b
6
+ metadata.gz: 72e629a77d117b4c370947e72eddc96ebddf766eff5d0459c5291074ab22033ded29e032563e00112051d0ef122e36bf4beca3e81ff94d9f5e9cd5c6576939a8
7
+ data.tar.gz: 61a4b4d3208796c7b4e365307fd527b081e5eb1be3fbebe02272883a3b7efafbb45c31234300dbe517f2eadc576b3cb4dc11fd91615f706704396541984c91bd
data/CHANGELOG.md CHANGED
@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ # [0.26.1] - 07-31-2024
11
+
12
+ - 0.26.0 didn't publish correctly. There's no meaningful difference between 0.26.1 and 0.26.0.
13
+
14
+ # [0.26.0] - 07-26-2024
15
+
16
+ - Adapter class now has `nodes` API in [830](https://github.com/OSC/ood_core/pull/830).
17
+ - QueueInfo objects are GPU aware in [825](https://github.com/OSC/ood_core/pull/825).
18
+ - Systemd adapter bugfix for zsh in [834](https://github.com/OSC/ood_core/pull/834).
19
+ - Websockify timeout is variable in [840](https://github.com/OSC/ood_core/pull/840).
20
+ - Slurm adapter now forces utf-8 encoding in [842](https://github.com/OSC/ood_core/pull/842).
21
+
10
22
  # [0.25.0] - 03-27-2024
11
23
 
12
24
  - [828](https://github.com/OSC/ood_core/pull/828) and [826](https://github.com/OSC/ood_core/pull/826)
@@ -535,6 +547,9 @@ Functionally the same as [0.17.3] but with some CI updates.
535
547
  - Initial release!
536
548
 
537
549
  [Unreleased]: https://github.com/OSC/ood_core/compare/v0.24.2...HEAD
550
+ [0.26.1]: https://github.com/OSC/ood_core/compare/v0.26.0...v0.26.1
551
+ [0.26.0]: https://github.com/OSC/ood_core/compare/v0.25.0...v0.26.0
552
+ [0.25.0]: https://github.com/OSC/ood_core/compare/v0.24.2...v0.25.0
538
553
  [0.24.2]: https://github.com/OSC/ood_core/compare/v0.24.1...v0.24.2
539
554
  [0.24.1]: https://github.com/OSC/ood_core/compare/v0.24.0...v0.24.1
540
555
  [0.24.0]: https://github.com/OSC/ood_core/compare/v0.23.5...v0.24.0
data/Rakefile CHANGED
@@ -1,6 +1,17 @@
1
1
  require "bundler/gem_tasks"
2
2
  require "rspec/core/rake_task"
3
+ require "minitest/test_task"
4
+
5
+ require_relative 'lib/tasks/slurm'
3
6
 
4
7
  RSpec::Core::RakeTask.new(:spec)
5
8
 
9
+
10
+ Minitest::TestTask.create(:test) do |t|
11
+ t.libs << "test"
12
+ t.libs << "lib"
13
+ t.warning = false
14
+ t.test_globs = ["test/**/*_test.rb"]
15
+ end
16
+
6
17
  task :default => :spec
@@ -233,6 +233,7 @@ module OodCore
233
233
 
234
234
  args.concat ["-N", script.job_name] unless script.job_name.nil?
235
235
  args.concat ["-o", script.output_path] unless script.output_path.nil?
236
+ args.concat ['--mpi', "proc=#{script.cores}"] unless script.cores.nil?
236
237
  if script.error_path.nil?
237
238
  args.concat ["-j"]
238
239
  else
@@ -239,7 +239,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
239
239
  node_selector = native_data[:node_selector].nil? ? {} : native_data[:node_selector]
240
240
  gpu_type = native_data[:gpu_type].nil? ? "nvidia.com/gpu" : native_data[:gpu_type]
241
241
 
242
- template = ERB.new(File.read(resource_file), nil, '-')
242
+ template = ERB.new(File.read(resource_file), trim_mode: '-')
243
243
 
244
244
  [template.result(binding), id]
245
245
  end
@@ -271,7 +271,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
271
271
  end
272
272
 
273
273
  def namespace
274
- "#{namespace_prefix}#{username}"
274
+ "#{namespace_prefix}#{username.gsub(/[.@]/, '-')}"
275
275
  end
276
276
 
277
277
  def formatted_ns_cmd
@@ -170,6 +170,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
170
170
  'script_timeout' => script_timeout(script),
171
171
  'session_name' => session_name,
172
172
  'singularity_bin' => singularity_bin,
173
+ 'singularity_options' => singularity_options(script.native),
173
174
  'singularity_image' => singularity_image(script.native),
174
175
  'ssh_hosts' => ssh_hosts,
175
176
  'tmux_bin' => tmux_bin,
@@ -205,6 +206,12 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
205
206
  native[:singularity_bindpath]
206
207
  end
207
208
 
209
+ def singularity_options(native)
210
+ return '' unless native && native[:singularity_options]
211
+
212
+ native[:singularity_options]
213
+ end
214
+
208
215
  def script_timeout(script)
209
216
  wall_time = script.wall_time.to_i
210
217
  return site_timeout if wall_time == 0
@@ -294,4 +301,4 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
294
301
  end.compact.last.to_s
295
302
  end
296
303
 
297
- end
304
+ end
@@ -47,7 +47,7 @@ trap exit_script SIGINT SIGTERM
47
47
  OUTPUT_PATH=<%= output_path %>
48
48
  ERROR_PATH=<%= error_path %>
49
49
  ({
50
- timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
50
+ timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= singularity_options %> <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
51
51
  } | tee "$OUTPUT_PATH") 3>&1 1>&2 2>&3 | tee "$ERROR_PATH"
52
52
 
53
53
  <%= email_on_terminated %>
@@ -91,6 +91,7 @@ class OodCore::Job::Adapters::Lsf::Helper
91
91
  args.concat ["-b", script.start_time.localtime.strftime("%Y:%m:%d:%H:%M")] unless script.start_time.nil?
92
92
  args.concat ["-W", (script.wall_time / 60).to_i] unless script.wall_time.nil?
93
93
  args.concat ["-L", script.shell_path.to_s] unless script.shell_path.nil?
94
+ args.concat ['-n', script.cores] unless script.cores.nil?
94
95
 
95
96
  # environment
96
97
  env = script.job_environment || {}
@@ -269,6 +269,7 @@ module OodCore
269
269
  args.concat ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
270
270
  args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
271
271
  args.concat ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
272
+ args.concat ppn(script)
272
273
 
273
274
  # Set dependencies
274
275
  depend = []
@@ -422,6 +423,13 @@ module OodCore
422
423
  '#PBS'
423
424
  end
424
425
 
426
+ # place holder for when we support both nodes and cpus.
427
+ def ppn(script)
428
+ return [] if script.cores.nil?
429
+
430
+ ['-l', "ncpus=#{script.cpus}"]
431
+ end
432
+
425
433
  private
426
434
  # Convert duration to seconds
427
435
  def duration_in_seconds(time)
@@ -372,10 +372,14 @@ module OodCore
372
372
  hsh[:AllowAccounts].to_s.split(',')
373
373
  end
374
374
 
375
-
376
375
  hsh[:deny_accounts] = hsh[:DenyAccounts].nil? ? [] : hsh[:DenyAccounts].to_s.split(',')
377
- hsh[:tres] = hsh[:TRES].nil? ? {} : hsh[:TRES].to_s.split(',').map { |str| str.split('=') }.to_h
378
376
 
377
+ hsh[:tres] = case hsh[:TRES]
378
+ when nil, '(null)', ''
379
+ {}
380
+ else
381
+ hsh[:TRES].to_s.split(',').map { |str| str.split('=') }.to_h
382
+ end
379
383
  OodCore::Job::QueueInfo.new(**hsh)
380
384
  end
381
385
 
@@ -526,6 +530,7 @@ module OodCore
526
530
  args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
527
531
  args.concat ['--qos', script.qos] unless script.qos.nil?
528
532
  args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
533
+ args.concat ['-n', script.cores] unless script.cores.nil?
529
534
  # ignore nodes, don't know how to do this for slurm
530
535
 
531
536
  # Set dependencies
@@ -10,7 +10,7 @@ require 'time'
10
10
  # @api private
11
11
  class OodCore::Job::Adapters::LinuxSystemd::Launcher
12
12
  attr_reader :debug, :site_timeout, :session_name_label, :ssh_hosts,
13
- :strict_host_checking, :username
13
+ :strict_host_checking, :username, :ssh_keyfile
14
14
  # The root exception class that all LinuxSystemd adapter-specific exceptions inherit
15
15
  # from
16
16
  class Error < StandardError; end
@@ -26,6 +26,7 @@ class OodCore::Job::Adapters::LinuxSystemd::Launcher
26
26
  ssh_hosts:,
27
27
  strict_host_checking: false,
28
28
  submit_host:,
29
+ ssh_keyfile: "",
29
30
  **_
30
31
  )
31
32
  @debug = !! debug
@@ -35,6 +36,7 @@ class OodCore::Job::Adapters::LinuxSystemd::Launcher
35
36
  @strict_host_checking = strict_host_checking
36
37
  @submit_host = submit_host
37
38
  @username = Etc.getlogin
39
+ @ssh_keyfile = ssh_keyfile
38
40
  end
39
41
 
40
42
  # @param hostname [#to_s] The hostname to submit the work to
@@ -97,27 +99,27 @@ class OodCore::Job::Adapters::LinuxSystemd::Launcher
97
99
  # if ! strict_host_checking
98
100
  # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
99
101
  # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
102
+ # if ssh_keyfile
103
+ # -i ssh_keyfile (Use this keyfile location)
100
104
  #
101
105
  # @param destination_host [#to_s] the destination host you wish to ssh into
102
106
  # @param cmd [Array<#to_s>] the command to be executed on the destination host
103
107
  def ssh_cmd(destination_host, cmd)
104
- if strict_host_checking
105
- [
106
- 'ssh', '-t',
107
- '-p', OodCore::Job::Adapters::Helper.ssh_port,
108
- '-o', 'BatchMode=yes',
109
- "#{username}@#{destination_host}"
110
- ].concat(cmd)
111
- else
112
- [
113
- 'ssh', '-t',
114
- '-p', OodCore::Job::Adapters::Helper.ssh_port,
115
- '-o', 'BatchMode=yes',
116
- '-o', 'UserKnownHostsFile=/dev/null',
117
- '-o', 'StrictHostKeyChecking=no',
118
- "#{username}@#{destination_host}"
119
- ].concat(cmd)
120
- end
108
+
109
+ sshcmd = [
110
+ 'ssh', '-t',
111
+ '-p', OodCore::Job::Adapters::Helper.ssh_port,
112
+ '-o', 'Batchmode=yes',
113
+ ]
114
+
115
+ sshcmd.concat([
116
+ '-o','StrictHostKeyChecking=no',
117
+ '-o','UserKnownHostsFile=/dev/null',
118
+ ]) unless strict_host_checking
119
+
120
+ sshcmd.concat(['-i',ssh_keyfile.to_s,]) unless ssh_keyfile.to_s.empty?
121
+
122
+ sshcmd.concat(["#{username}@#{destination_host}"]).concat(cmd)
121
123
  end
122
124
 
123
125
  def shell
@@ -15,6 +15,7 @@ module OodCore
15
15
  # @option config [Object] :ssh_hosts (nil) The list of permissable hosts, defaults to :submit_host
16
16
  # @option config [Object] :strict_host_checking (true) Set to false to disable strict host checking and updating the known_hosts file
17
17
  # @option config [Object] :submit_host The SSH target to connect to, may be the head of a round-robin
18
+ # @option config [Object] :ssh_keyfile The SSH Key file to use as identity.
18
19
  def self.build_systemd(config)
19
20
  c = config.to_h.symbolize_keys
20
21
  debug = c.fetch(:debug, false)
@@ -22,6 +23,7 @@ module OodCore
22
23
  ssh_hosts = c.fetch(:ssh_hosts, [c[:submit_host]])
23
24
  strict_host_checking = c.fetch(:strict_host_checking, true)
24
25
  submit_host = c[:submit_host]
26
+ ssh_keyfile = c.fetch(:ssh_keyfile, "")
25
27
 
26
28
  Adapters::LinuxSystemd.new(
27
29
  ssh_hosts: ssh_hosts,
@@ -31,6 +33,7 @@ module OodCore
31
33
  ssh_hosts: ssh_hosts,
32
34
  strict_host_checking: strict_host_checking,
33
35
  submit_host: submit_host,
36
+ ssh_keyfile: ssh_keyfile,
34
37
  )
35
38
  )
36
39
  end
@@ -160,6 +160,7 @@ module OodCore
160
160
  args.concat ['-t', script.job_array_request] unless script.job_array_request.nil?
161
161
  args.concat ['-l', "qos=#{script.qos}"] unless script.qos.nil?
162
162
  args.concat ['-l', "gpus=#{script.gpus_per_node}"] unless script.gpus_per_node.nil?
163
+ args.concat ppn(script)
163
164
 
164
165
  # Set environment variables
165
166
  env = script.job_environment.to_h
@@ -302,6 +303,13 @@ module OodCore
302
303
  '#QSUB'
303
304
  end
304
305
 
306
+ # place holder for when we support both nodes and cpus.
307
+ def ppn(script)
308
+ return [] if script.cores.nil?
309
+
310
+ ['-l', "procs=#{script.cpus}"]
311
+ end
312
+
305
313
  private
306
314
  # Convert duration to seconds
307
315
  def duration_in_seconds(time)
@@ -108,6 +108,10 @@ module OodCore
108
108
  # @return [Integer, nil] gpus per node
109
109
  attr_reader :gpus_per_node
110
110
 
111
+ # The core request for this job
112
+ # @return [Integer, nil] cores
113
+ attr_reader :cores
114
+
111
115
  # Object detailing any native specifications that are implementation specific
112
116
  # @note Should not be used at all costs.
113
117
  # @return [Object, nil] native specifications
@@ -151,7 +155,8 @@ module OodCore
151
155
  output_path: nil, error_path: nil, reservation_id: nil,
152
156
  queue_name: nil, priority: nil, start_time: nil,
153
157
  wall_time: nil, accounting_id: nil, job_array_request: nil,
154
- qos: nil, gpus_per_node: nil, native: nil, copy_environment: nil, **_)
158
+ qos: nil, gpus_per_node: nil, native: nil, copy_environment: nil,
159
+ cores: nil, **_)
155
160
  @content = content.to_s
156
161
 
157
162
  @submit_as_hold = submit_as_hold
@@ -179,6 +184,7 @@ module OodCore
179
184
  @gpus_per_node = gpus_per_node && gpus_per_node.to_i
180
185
  @native = native
181
186
  @copy_environment = (copy_environment.nil?) ? nil : !! copy_environment
187
+ @cores = cores&.to_i
182
188
  end
183
189
 
184
190
  # Convert object to hash
@@ -209,7 +215,8 @@ module OodCore
209
215
  qos: qos,
210
216
  gpus_per_node: gpus_per_node,
211
217
  native: native,
212
- copy_environment: copy_environment
218
+ cores: cores,
219
+ copy_environment: copy_environment,
213
220
  }
214
221
  end
215
222
 
@@ -1,4 +1,4 @@
1
1
  module OodCore
2
2
  # The current version of {OodCore}
3
- VERSION = "0.26.1"
3
+ VERSION = "0.27.1"
4
4
  end
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../ood_core'
4
+ require_relative '../ood_core/job/adapters/slurm'
5
+
6
+ namespace :slurm do
7
+
8
+ desc 'Get squeue output in the format this gem expects'
9
+ task :squeue do
10
+ fields = OodCore::Job::Adapters::Slurm::Batch.new.all_squeue_fields
11
+ args = OodCore::Job::Adapters::Slurm::Batch.new.squeue_args(options: fields.values)
12
+
13
+ single_job = `squeue #{args.join(' ')}`.split("\n")[0...2]
14
+
15
+ puts single_job
16
+ end
17
+ end
data/ood_core.gemspec CHANGED
@@ -31,4 +31,6 @@ Gem::Specification.new do |spec|
31
31
  spec.add_development_dependency "pry", "~> 0.10"
32
32
  spec.add_development_dependency "timecop", "~> 0.8"
33
33
  spec.add_development_dependency "climate_control", "~> 1.2.0"
34
+ spec.add_development_dependency "minitest", "~> 5"
35
+ spec.add_development_dependency "mocha", "~> 2.4"
34
36
  end
metadata CHANGED
@@ -1,16 +1,16 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ood_core
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.26.1
4
+ version: 0.27.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Franz
8
8
  - Morgan Rodgers
9
9
  - Jeremy Nicklas
10
- autorequire:
10
+ autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2024-07-31 00:00:00.000000000 Z
13
+ date: 2025-02-10 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: ood_support
@@ -138,6 +138,34 @@ dependencies:
138
138
  - - "~>"
139
139
  - !ruby/object:Gem::Version
140
140
  version: 1.2.0
141
+ - !ruby/object:Gem::Dependency
142
+ name: minitest
143
+ requirement: !ruby/object:Gem::Requirement
144
+ requirements:
145
+ - - "~>"
146
+ - !ruby/object:Gem::Version
147
+ version: '5'
148
+ type: :development
149
+ prerelease: false
150
+ version_requirements: !ruby/object:Gem::Requirement
151
+ requirements:
152
+ - - "~>"
153
+ - !ruby/object:Gem::Version
154
+ version: '5'
155
+ - !ruby/object:Gem::Dependency
156
+ name: mocha
157
+ requirement: !ruby/object:Gem::Requirement
158
+ requirements:
159
+ - - "~>"
160
+ - !ruby/object:Gem::Version
161
+ version: '2.4'
162
+ type: :development
163
+ prerelease: false
164
+ version_requirements: !ruby/object:Gem::Requirement
165
+ requirements:
166
+ - - "~>"
167
+ - !ruby/object:Gem::Version
168
+ version: '2.4'
141
169
  description: Open OnDemand core library that provides support for an HPC Center to
142
170
  globally define HPC services that web applications can then take advantage of.
143
171
  email:
@@ -221,12 +249,13 @@ files:
221
249
  - lib/ood_core/refinements/drmaa_extensions.rb
222
250
  - lib/ood_core/refinements/hash_extensions.rb
223
251
  - lib/ood_core/version.rb
252
+ - lib/tasks/slurm.rb
224
253
  - ood_core.gemspec
225
254
  homepage: https://github.com/OSC/ood_core
226
255
  licenses:
227
256
  - MIT
228
257
  metadata: {}
229
- post_install_message:
258
+ post_install_message:
230
259
  rdoc_options: []
231
260
  require_paths:
232
261
  - lib
@@ -242,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
242
271
  version: '0'
243
272
  requirements: []
244
273
  rubygems_version: 3.1.2
245
- signing_key:
274
+ signing_key:
246
275
  specification_version: 4
247
276
  summary: Open OnDemand core library
248
277
  test_files: []