ood_core 0.25.0 → 0.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad05a0d8cf307f0237329f0e33e8a66f809781e5b164ddf20b9f03d65078451d
-  data.tar.gz: 396dde55b4b5aaa321ccaf6fbc3cb049d01239335533c35c39ee9a11a1aa5d9b
+  metadata.gz: 6aafe5b9d84b0b825241691c28be26cbce7402534e0ac1555a033191d13a7a1a
+  data.tar.gz: f214cbaf84997e4c27dc184f3b3a12cfc82e510c5d6eb7b9ccdfb5df0b164089
 SHA512:
-  metadata.gz: 4a1d7cb53117266c577eaa7792d867508f3e94ef78f2f9af7817882ad1f6165d5d43e8179e17163a83dd438627215700501ea75281204ee6b7d900649c77de9f
-  data.tar.gz: 4aaf01f40d43e7b496a053e2ef85548769beb169c78e954cd055659ecc08f2e8ef7221561f94a184eb986b9f13d52c2efda30ffdd07faeb697fab4d9d4411ce7
+  metadata.gz: d1cc480563eaffcafeaf68b09ec55e5cec02d2c610b7264e45c0c0f0d830ac11e62a9b91fe395a00140c38b8bc5b01390e52d4dc8d962579fee0cf7ff33c6fc3
+  data.tar.gz: 68d5c95f5a4d87ab89cd5938d92c2d186398701942fd8a460409e5f5864b9eccedcb8ac8668d211dc0318886e6978aa0528371ede004466f0b9dcf7bc2a95118

data/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+# [0.26.1] - 07-31-2024
+
+- 0.26.0 didn't publish correctly. There's no meaningful difference between 0.26.1 and 0.26.0.
+
+# [0.26.0] - 07-26-2024
+
+- Adapter class now has `nodes` API in [830](https://github.com/OSC/ood_core/pull/830).
+- QueueInfo objects are GPU aware in [825](https://github.com/OSC/ood_core/pull/825).
+- Systemd adapter bugfix for zsh in [834](https://github.com/OSC/ood_core/pull/834).
+- Websockify timeout is variable in [840](https://github.com/OSC/ood_core/pull/840).
+- Slurm adapter now forces utf-8 encoding in [842](https://github.com/OSC/ood_core/pull/842).
+
 # [0.25.0] - 03-27-2024
 
 - [828](https://github.com/OSC/ood_core/pull/828) and [826](https://github.com/OSC/ood_core/pull/826)
@@ -535,6 +547,9 @@ Functionally the same as [0.17.3] but with some CI updates.
 - Initial release!
 
 [Unreleased]: https://github.com/OSC/ood_core/compare/v0.24.2...HEAD
+[0.26.1]: https://github.com/OSC/ood_core/compare/v0.26.0...v0.26.1
+[0.26.0]: https://github.com/OSC/ood_core/compare/v0.25.0...v0.26.0
+[0.25.0]: https://github.com/OSC/ood_core/compare/v0.24.2...v0.25.0
 [0.24.2]: https://github.com/OSC/ood_core/compare/v0.24.1...v0.24.2
 [0.24.1]: https://github.com/OSC/ood_core/compare/v0.24.0...v0.24.1
 [0.24.0]: https://github.com/OSC/ood_core/compare/v0.23.5...v0.24.0

data/Rakefile

@@ -1,6 +1,17 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "minitest/test_task"
+
+require_relative 'lib/tasks/slurm'
 
 RSpec::Core::RakeTask.new(:spec)
 
+
+Minitest::TestTask.create(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.warning = false
+  t.test_globs = ["test/**/*_test.rb"]
+end
+
 task :default => :spec

data/lib/ood_core/batch_connect/templates/vnc.rb

@@ -134,7 +134,8 @@ module OodCore
           def after_script
             websockify_cmd = context.fetch(:websockify_cmd, "${WEBSOCKIFY_CMD:-/opt/websockify/run}").to_s
             websockify_hb = context.fetch(:websockify_heartbeat_seconds, "${WEBSOCKIFY_HEARTBEAT_SECONDS:-30}").to_s
-            
+            websockify_timeout_seconds = context.fetch(:websockify_timeout_seconds, '${WEBSOCKIFY_TIMEOUT_SECONDS:-10}').to_s
+
             <<-EOT.gsub(/^ {14}/, "")
               #{super}
 
@@ -146,6 +147,7 @@ module OodCore
                 #{websockify_cmd} $1 --heartbeat=#{websockify_hb} $2 &> $log_file &
                 local ws_pid=$!
                 local counter=0
+                local max_timeout=#{websockify_timeout_seconds}
 
                 # wait till websockify has successfully started
                 echo "[websockify]: pid: $ws_pid (proxying $1 ==> $2)" >&2
@@ -156,9 +158,9 @@ module OodCore
                   if ! ps $ws_pid > /dev/null; then
                     echo "[websockify]: failed to launch!" >&2
                     return 1
-                  elif [ $counter -ge 5 ]; then
-                    # timeout after ~5 seconds
-                    echo "[websockify]: timed-out :(!" >&2
+                  elif [ $counter -ge $max_timeout ]; then
+                    # timeout after max_timeout seconds
+                    echo "[websockify]: timed-out after $max_timeout seconds :(!" >&2
                     return 1
                   else
                     sleep 1

data/lib/ood_core/job/adapter.rb

@@ -213,6 +213,14 @@ module OodCore
       def queues
         []
       end
+
+      # Return the list of nodes for this scheduler.
+      #
+      # Subclasses that do not implement this will return empty arrays.
+      # @return [Array<NodeInfo>]
+      def nodes
+        []
+      end
     end
   end
 end

data/lib/ood_core/job/adapters/fujitsu_tcs.rb

@@ -233,6 +233,7 @@ module OodCore
 
           args.concat ["-N", script.job_name] unless script.job_name.nil?
           args.concat ["-o", script.output_path] unless script.output_path.nil?
+          args.concat ['--mpi', "proc=#{script.cores}"] unless script.cores.nil?
           if script.error_path.nil?
             args.concat ["-j"]
           else

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -239,7 +239,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     node_selector = native_data[:node_selector].nil? ? {} : native_data[:node_selector]
     gpu_type = native_data[:gpu_type].nil? ? "nvidia.com/gpu" : native_data[:gpu_type]
 
-    template = ERB.new(File.read(resource_file), nil, '-')
+    template = ERB.new(File.read(resource_file), trim_mode: '-')
 
     [template.result(binding), id]
   end
@@ -271,7 +271,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def namespace
-    "#{namespace_prefix}#{username}"
+    "#{namespace_prefix}#{username.gsub(/[.@]/, '-')}"
   end
 
   def formatted_ns_cmd

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -170,6 +170,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'script_timeout' => script_timeout(script),
         'session_name' => session_name,
         'singularity_bin' => singularity_bin,
+        'singularity_options' => singularity_options(script.native),
         'singularity_image' => singularity_image(script.native),
         'ssh_hosts' => ssh_hosts,
         'tmux_bin' => tmux_bin,
@@ -205,6 +206,12 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     native[:singularity_bindpath]
   end
 
+  def singularity_options(native)
+    return '' unless native && native[:singularity_options]
+
+    native[:singularity_options]
+  end
+
   def script_timeout(script)
     wall_time = script.wall_time.to_i
     return site_timeout if wall_time == 0
@@ -294,4 +301,4 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     end.compact.last.to_s
   end
 
-end
+end

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -47,7 +47,7 @@ trap exit_script SIGINT SIGTERM
 OUTPUT_PATH=<%= output_path %>
 ERROR_PATH=<%= error_path %>
 ({
-timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
+timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= singularity_options %> <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
 } | tee "$OUTPUT_PATH") 3>&1 1>&2 2>&3 | tee "$ERROR_PATH"
 
 <%= email_on_terminated %>

data/lib/ood_core/job/adapters/lsf/helper.rb

@@ -91,6 +91,7 @@ class OodCore::Job::Adapters::Lsf::Helper
     args.concat ["-b", script.start_time.localtime.strftime("%Y:%m:%d:%H:%M")] unless script.start_time.nil?
     args.concat ["-W", (script.wall_time / 60).to_i] unless script.wall_time.nil?
     args.concat ["-L", script.shell_path.to_s] unless script.shell_path.nil?
+    args.concat ['-n', script.cores] unless script.cores.nil?
 
     # environment
     env = script.job_environment || {}

data/lib/ood_core/job/adapters/pbspro.rb

@@ -269,6 +269,7 @@ module OodCore
           args.concat ["-a", script.start_time.localtime.strftime("%C%y%m%d%H%M.%S")] unless script.start_time.nil?
           args.concat ["-A", script.accounting_id] unless script.accounting_id.nil?
           args.concat ["-l", "walltime=#{seconds_to_duration(script.wall_time)}"] unless script.wall_time.nil?
+          args.concat ppn(script)
 
           # Set dependencies
           depend = []
@@ -422,6 +423,13 @@ module OodCore
           '#PBS'
         end
 
+        # place holder for when we support both nodes and cpus.
+        def ppn(script)
+          return [] if script.cores.nil?
+
+          ['-l', "ncpus=#{script.cpus}"]
+        end
+
         private
           # Convert duration to seconds
           def duration_in_seconds(time)

data/lib/ood_core/job/adapters/slurm.rb

@@ -41,7 +41,7 @@ module OodCore
         # calculated from gres string
         # @return [Integer] the number of gpus in gres
         def self.gpus_from_gres(gres)
-          gres.to_s.scan(/gpu:[^,]*(\d+)/).flatten.map(&:to_i).sum
+          gres.to_s.scan(/gpu[^(,]*[:=](\d+)/).flatten.map(&:to_i).sum
         end
 
         # Object used for simplified communication with a Slurm batch server
@@ -169,6 +169,7 @@ module OodCore
                 # jobs << job
                 #
                 # assuming keys and values are same length! if not we have an error!
+                line = line.encode('UTF-8', invalid: :replace, undef: :replace)
                 values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR)
                 jobs << Hash[fields.keys.zip(values)] unless values.empty?
               end
@@ -327,13 +328,37 @@ module OodCore
 
             [].tap do |ret_arr|
               info_raw.each_line do |line|
-                ret_arr << str_to_acct_info(line)
+                ret_arr << str_to_queue_info(line)
               end
             end
           end
 
+          def all_sinfo_node_fields
+            {
+              procs: '%c',
+              name: '%n',
+              features: '%f'
+            }
+          end
+
+          def nodes
+            args = all_sinfo_node_fields.values.join(UNIT_SEPARATOR)
+            output = call('sinfo', '-ho', "#{RECORD_SEPARATOR}#{args}")
+
+            output.each_line(RECORD_SEPARATOR).map do |line|
+              values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR)
+
+              next if values.empty?
+
+              data = Hash[all_sinfo_node_fields.keys.zip(values)]
+              data[:name] = data[:name].to_s.split(',').first
+              data[:features] = data[:features].to_s.split(',')
+              NodeInfo.new(**data)
+            end.compact
+          end
+
           private
-            def str_to_acct_info(line)
+            def str_to_queue_info(line)
               hsh = line.split(' ').map do |token|
                 m = token.match(/^(?<key>\w+)=(?<value>.+)$/)
                 [m[:key], m[:value]]
@@ -349,6 +374,7 @@ module OodCore
 
 
               hsh[:deny_accounts] = hsh[:DenyAccounts].nil? ? [] : hsh[:DenyAccounts].to_s.split(',')
+              hsh[:tres] = hsh[:TRES].nil? ? {} : hsh[:TRES].to_s.split(',').map { |str| str.split('=') }.to_h
 
               OodCore::Job::QueueInfo.new(**hsh)
             end
@@ -500,6 +526,7 @@ module OodCore
           args.concat ['-a', script.job_array_request] unless script.job_array_request.nil?
           args.concat ['--qos', script.qos] unless script.qos.nil?
           args.concat ['--gpus-per-node', script.gpus_per_node] unless script.gpus_per_node.nil?
+          args.concat ['-n', script.cores] unless script.cores.nil?
           # ignore nodes, don't know how to do this for slurm
 
           # Set dependencies
@@ -669,6 +696,10 @@ module OodCore
           @slurm.queues
         end
 
+        def nodes
+          @slurm.nodes
+        end
+
         private
           # Convert duration to seconds
           def duration_in_seconds(time)

data/lib/ood_core/job/adapters/systemd/launcher.rb

@@ -204,7 +204,7 @@ class OodCore::Job::Adapters::LinuxSystemd::Launcher
 
   # List all Systemd sessions on destination_host started by this adapter
   def list_remote_systemd_session(destination_host)
-    cmd = ssh_cmd(destination_host, ['systemctl', '--user', 'show', '-t', 'service', '--state=running', "#{session_name_label}-*"])
+    cmd = ssh_cmd(destination_host, ['systemctl', '--user', 'show', '-t', 'service', '--state=running', "#{session_name_label}-\\*"])
 
     # individual units are separated with an empty line
     call(*cmd).split("\n\n").map do |oneunit|

data/lib/ood_core/job/adapters/torque.rb

@@ -160,6 +160,7 @@ module OodCore
             args.concat ['-t', script.job_array_request] unless script.job_array_request.nil?
             args.concat ['-l', "qos=#{script.qos}"] unless script.qos.nil?
             args.concat ['-l', "gpus=#{script.gpus_per_node}"] unless script.gpus_per_node.nil?
+            args.concat ppn(script)
 
             # Set environment variables
             env = script.job_environment.to_h
@@ -302,6 +303,13 @@ module OodCore
           '#QSUB'
         end
 
+        # place holder for when we support both nodes and cpus.
+        def ppn(script)
+          return [] if script.cores.nil?
+
+          ['-l', "procs=#{script.cpus}"]
+        end
+
         private
           # Convert duration to seconds
           def duration_in_seconds(time)

data/lib/ood_core/job/node_info.rb

@@ -10,17 +10,26 @@ module OodCore
       # @return [Integer, nil] number of procs
       attr_reader :procs
 
+      # The features associated with this node.
+      # @return [Array<String>, []]
+      attr_reader :features
+
       # @param name [#to_s] node name
       # @param procs [#to_i, nil] number of procs
-      def initialize(name:, procs: nil, **_)
+      # @param features [#to_a, []] list of features
+      def initialize(name:, procs: nil, features: [], **_)
         @name  = name.to_s
         @procs = procs && procs.to_i
+        @features = features.to_a
       end
 
       # Convert object to hash
       # @return [Hash] object as hash
       def to_h
-        { name: name, procs: procs }
+        instance_variables.map do |var|
+          name = var.to_s.gsub('@', '').to_sym
+          [name, send(name)]
+        end.to_h
       end
 
       # The comparison operator

data/lib/ood_core/job/queue_info.rb

@@ -20,9 +20,13 @@ class OodCore::Job::QueueInfo
   # The accounts that are not allowed to use this queue.
   attr_reader :deny_accounts
 
+  # An Hash of Trackable Resources and their values.
+  attr_reader :tres
+
   def initialize(**opts)
     @name = opts.fetch(:name, 'unknown')
     @qos = opts.fetch(:qos, [])
+    @tres = opts.fetch(:tres, {})
 
     allow_accounts = opts.fetch(:allow_accounts, nil)
     @allow_accounts = if allow_accounts.nil?
@@ -42,4 +46,8 @@ class OodCore::Job::QueueInfo
       [name, send(name)]
     end.to_h
   end
+
+  def gpu?
+    tres.keys.any? { |name| name.to_s.match?(%r{^gres/gpu($|:)}i) }
+  end
 end

data/lib/ood_core/job/script.rb

@@ -108,6 +108,10 @@ module OodCore
       # @return [Integer, nil] gpus per node
       attr_reader :gpus_per_node
 
+      # The core request for this job
+      # @return [Integer, nil] cores
+      attr_reader :cores
+
       # Object detailing any native specifications that are implementation specific
       # @note Should not be used at all costs.
       # @return [Object, nil] native specifications
@@ -151,7 +155,8 @@ module OodCore
                      output_path: nil, error_path: nil, reservation_id: nil,
                      queue_name: nil, priority: nil, start_time: nil,
                      wall_time: nil, accounting_id: nil, job_array_request: nil,
-                     qos: nil, gpus_per_node: nil, native: nil, copy_environment: nil, **_)
+                     qos: nil, gpus_per_node: nil, native: nil, copy_environment: nil,
+                     cores: nil, **_)
         @content = content.to_s
 
         @submit_as_hold      = submit_as_hold
@@ -179,6 +184,7 @@ module OodCore
         @gpus_per_node      = gpus_per_node     && gpus_per_node.to_i
         @native             = native
         @copy_environment   = (copy_environment.nil?) ? nil : !! copy_environment
+        @cores              = cores&.to_i
       end
 
       # Convert object to hash
@@ -209,7 +215,8 @@ module OodCore
           qos:                 qos,
           gpus_per_node:       gpus_per_node,
           native:              native,
-          copy_environment:    copy_environment
+          cores:               cores,
+          copy_environment:    copy_environment,
         }
       end
 

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.25.0"
+  VERSION = "0.27.0"
 end

data/lib/tasks/slurm.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../ood_core'
+require_relative '../ood_core/job/adapters/slurm'
+
+namespace :slurm do
+
+  desc 'Get squeue output in the format this gem expects'
+  task :squeue do
+    fields = OodCore::Job::Adapters::Slurm::Batch.new.all_squeue_fields
+    args = OodCore::Job::Adapters::Slurm::Batch.new.squeue_args(options: fields.values)
+
+    single_job = `squeue #{args.join(' ')}`.split("\n")[0...2]
+
+    puts single_job
+  end
+end

data/ood_core.gemspec

@@ -23,12 +23,14 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = ">= 2.5.0"
 
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
-  spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
+  spec.add_runtime_dependency "ffi", "~> 1.16.3"
   spec.add_runtime_dependency "rexml", "~> 3.2"
   spec.add_development_dependency "bundler", "~> 2.1"
-  spec.add_development_dependency "rake", "~> 13.1.0"
+  spec.add_development_dependency "rake", "~> 13.2.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"
   spec.add_development_dependency "timecop", "~> 0.8"
   spec.add_development_dependency "climate_control", "~> 1.2.0"
+  spec.add_development_dependency "minitest", "~> 5"
+  spec.add_development_dependency "mocha", "~> 2.4"
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.25.0
+  version: 0.27.0
 platform: ruby
 authors:
 - Eric Franz
 - Morgan Rodgers
 - Jeremy Nicklas
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -32,20 +32,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.9.6
+        version: 1.16.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.9'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.9.6
+        version: 1.16.3
 - !ruby/object:Gem::Dependency
   name: rexml
   requirement: !ruby/object:Gem::Requirement
@@ -80,14 +74,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 13.1.0
+        version: 13.2.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -144,6 +138,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 description: Open OnDemand core library that provides support for an HPC Center to
   globally define HPC services that web applications can then take advantage of.
 email:
@@ -227,12 +249,13 @@ files:
 - lib/ood_core/refinements/drmaa_extensions.rb
 - lib/ood_core/refinements/hash_extensions.rb
 - lib/ood_core/version.rb
+- lib/tasks/slurm.rb
 - ood_core.gemspec
 homepage: https://github.com/OSC/ood_core
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -248,7 +271,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Open OnDemand core library
 test_files: []