ood_core 0.19.0 → 0.20.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f8d595b211ea1f26d22bac6d441ddc56a1dd4dc46a8439be2d60c4fdb1545b9
-  data.tar.gz: c04312cea53252aa4cac76a1dbccab6d40f974321630f5ac97931145bb404b0d
+  metadata.gz: 2c4e013f80e987d4d1cefbc78cc76bcff52e4083e0b84192b42807ae46806946
+  data.tar.gz: c4a1607904baccc1b063916ecf8e5a9692a9c0102a0d8cda3a9edf0ae760191f
 SHA512:
-  metadata.gz: c5e4d7da375953bf188f09f9124380e62a7599c43c2d758bb3fa14a7f33d397b7134cb1b18e1f9844347f2b9e375e0d864e73b6635e563cc2d127ab435eaff1c
-  data.tar.gz: a2f76e1121289d445f3666c6430420d805dede8c49ceb60a79de6b891b9903ded50f0cbca623f2354f63cf3de72d65cfef9b31ae08762b999888368778ca3a37
+  metadata.gz: ab3333366fc7802d59a15dead3b21e863d0017385053eea629a109a076c6e768ed1575378a34e68bb6c163b050be87a9cf323f087d02e4e2be4d349550bf5531
+  data.tar.gz: 234c13fbbc428717532bd93ba4e977cfd825e480c69daf66c737165ed7c5d8a951c329ced0312d525efc0b70cb4d11234c016c6216c2bb7f74573de854340889

data/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.20.2] - 07-28-2022
+
+- Fixed an issue with Slurm's `cluster_info` in [762](https://github.com/OSC/ood_core/pull/762).
+- Relaxed Ruby requirement down to 2.5 in [771](https://github.com/OSC/ood_core/pull/771).
+
+## [0.20.1] - 07-21-2022
+
+- Fixed turbovnc compatability issue with the -nohttpd flag in [767](https://github.com/OSC/ood_core/pull/767).
+
+## [0.20.0] - 06-03-2022
+
+- Adapters can now respond to `cluster_info` in [752](https://github.com/OSC/ood_core/pull/752). This returns information about the cluster like how many nodes are available and so on. Only Slurm support in this release.
+-  `OodCore::Job::Info` now has a `gpus` attribute in [753](https://github.com/OSC/ood_core/pull/753). Only Slurm support in this release.
+- Support Ruby 3 in [759](https://github.com/OSC/ood_core/pull/759)
+
 ## [0.19.0] - 02-03-2022
 
 ### Added
@@ -422,7 +437,10 @@ Functionally the same as [0.17.3] but with some CI updates.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.19.0...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.20.2...HEAD
+[0.20.2]: https://github.com/OSC/ood_core/compare/v0.20.1...v0.20.2
+[0.20.1]: https://github.com/OSC/ood_core/compare/v0.20.0...v0.20.1
+[0.20.0]: https://github.com/OSC/ood_core/compare/v0.19.0...v0.20.0
 [0.19.0]: https://github.com/OSC/ood_core/compare/v0.18.1...v0.19.0
 [0.18.1]: https://github.com/OSC/ood_core/compare/v0.18.0...v0.18.1
 [0.18.0]: https://github.com/OSC/ood_core/compare/v0.17.8...v0.18.0

data/lib/ood_core/batch_connect/templates/vnc.rb

@@ -86,8 +86,13 @@ module OodCore
                 # Clean up any old VNC sessions that weren't cleaned before
                 #{vnc_clean}
 
+                # for turbovnc 3.0 compatability.
+                if timeout 2 vncserver --help 2>&1 | grep 'nohttpd' >/dev/null 2>&1; then
+                  HTTPD_OPT='-nohttpd'
+                fi
+
                 # Attempt to start VNC server
-                VNC_OUT=$(vncserver -log "#{vnc_log}" -rfbauth "#{vnc_passwd}" -nohttpd -noxstartup #{vnc_args} 2>&1)
+                VNC_OUT=$(vncserver -log "#{vnc_log}" -rfbauth "#{vnc_passwd}" $HTTPD_OPT -noxstartup #{vnc_args} 2>&1)
                 VNC_PID=$(pgrep -s 0 Xvnc) # the script above will daemonize the Xvnc process
                 echo "${VNC_OUT}"
 

data/lib/ood_core/job/adapter.rb

@@ -33,6 +33,15 @@ module OodCore
         raise NotImplementedError, "subclass did not define #submit"
       end
 
+      # Retrieve the number of active and total cpus, nodes, and gpus
+      # @abstract Subclass is expected to implement {#cluster_stats}
+      # @raise [NotImplementedError] if subclass did not define {#cluster_stats}
+      # @return [ClusterInfo] Object containing quantified statistics about the
+      #   cluster's active/total cpus, nodes, and gpus
+      def cluster_info
+        raise NotImplementedError, "subclass did not define #cluster_stats"
+      end
+
       # Retrieve info for all jobs from the resource manager
       # @abstract Subclass is expected to implement {#info_all}
       # @raise [NotImplementedError] if subclass did not define {#info_all}

data/lib/ood_core/job/adapters/ccq.rb

@@ -228,7 +228,7 @@ module OodCore
           data_hash[:submission_time] = raw['dateSubmitted'].to_i
           data_hash[:queue_name] = raw['criteriaPriority']
 
-          Info.new(data_hash)
+          Info.new(**data_hash)
         end
 
         # extended data is just lines of 'key: value' value, so parse
@@ -242,7 +242,7 @@ module OodCore
 
           data.to_s.lines.drop(1).each do |line|
             match_data = ccqstat_regex.match(line)
-            infos << Info.new(ccqstat_match_to_hash(match_data)) if valid_ccqstat_match?(match_data)
+            infos << Info.new(**ccqstat_match_to_hash(match_data)) if valid_ccqstat_match?(match_data)
           end
 
           infos

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -93,7 +93,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
 
   def info(id)
     pod_json = safe_call('get', 'pod', id)
-    return OodCore::Job::Info.new({ id: id, status: 'completed' }) if pod_json.empty?
+    return OodCore::Job::Info.new(**{ id: id, status: 'completed' }) if pod_json.empty?
 
     service_json = safe_call('get', 'service', service_name(id))
     secret_json = safe_call('get', 'secret', secret_name(id))

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -31,7 +31,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
 
     pod_hash.deep_merge!(service_hash)
     pod_hash.deep_merge!(secret_hash)
-    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(pod_hash)
+    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(**pod_hash)
   rescue NoMethodError
     raise K8sDataError, "unable to read data correctly from json"
   end

data/lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb

@@ -2,8 +2,8 @@
 class OodCore::Job::Adapters::Kubernetes::K8sJobInfo < OodCore::Job::Info
   attr_reader :ood_connection_info
 
-  def initialize(ood_connection_info: {}, **options)
-    super(options)
-    @ood_connection_info = ood_connection_info
+  def initialize(options)
+    super(**options)
+    @ood_connection_info = options[:ood_connection_info]
   end
 end

data/lib/ood_core/job/adapters/lsf.rb

@@ -16,7 +16,7 @@ module OodCore
       # @option config [#to_h] :bin_overrides ({}) Optional overrides to LSF client executables
       # @option config [#to_s] :submit_host ('') Host to submit commands to
       def self.build_lsf(config)
-        batch = Adapters::Lsf::Batch.new(config.to_h.symbolize_keys)
+        batch = Adapters::Lsf::Batch.new(**config.to_h.symbolize_keys)
         Adapters::Lsf.new(batch: batch)
       end
     end

data/lib/ood_core/job/adapters/slurm.rb

@@ -36,6 +36,13 @@ module OodCore
         using Refinements::HashExtensions
         using Refinements::ArrayExtensions
 
+        # Get integer representing the number of gpus used by a node or job,
+        # calculated from gres string
+        # @return [Integer] the number of gpus in gres
+        def self.gpus_from_gres(gres)
+          gres.to_s.scan(/gpu:[^,]*(\d+)/).flatten.map(&:to_i).sum
+        end
+
         # Object used for simplified communication with a Slurm batch server
         # @api private
         class Batch
@@ -98,6 +105,22 @@ module OodCore
             @strict_host_checking = strict_host_checking
           end
 
+          # Get a ClusterInfo object containing information about the given cluster
+          # @return [ClusterInfo] object containing cluster details
+          def get_cluster_info
+            node_cpu_info = call("sinfo", "-aho %A/%D/%C").strip.split('/')
+            gres_length = call("sinfo", "-o %G").lines.map(&:strip).map(&:length).max + 2
+            gres_lines = call("sinfo", "-ahNO ,nodehost,gres:#{gres_length},gresused:#{gres_length}")
+                         .lines.uniq.map(&:split)
+            ClusterInfo.new(active_nodes: node_cpu_info[0].to_i,
+                            total_nodes: node_cpu_info[2].to_i,
+                            active_processors: node_cpu_info[3].to_i,
+                            total_processors: node_cpu_info[6].to_i,
+                            active_gpus: gres_lines.sum { |line| Slurm.gpus_from_gres(line[2]) },
+                            total_gpus: gres_lines.sum { |line| Slurm.gpus_from_gres(line[1]) }
+            )
+          end
+
           # Get a list of hashes detailing each of the jobs on the batch server
           # @example Status info for all jobs
           #   my_batch.get_jobs
@@ -454,6 +477,12 @@ module OodCore
           raise JobAdapterError, e.message
         end
 
+        # Retrieve info about active and total cpus, gpus, and nodes
+        # @return [Hash] information about cluster usage
+        def cluster_info
+          @slurm.get_cluster_info
+        end
+
         # Retrieve info for all jobs from the resource manager
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
@@ -643,7 +672,8 @@ module OodCore
               cpu_time: nil,
               submission_time: v[:submit_time] ? Time.parse(v[:submit_time]) : nil,
               dispatch_time: (v[:start_time].nil? || v[:start_time] == "N/A") ? nil : Time.parse(v[:start_time]),
-              native: v
+              native: v,
+              gpus: self.class.gpus_from_gres(v[:gres])
             )
           end
 

data/lib/ood_core/job/cluster_info.rb

@@ -0,0 +1,32 @@
+module OodCore
+  module Job
+    # An object that contains details about the cluster's active and total nodes, processors, and gpus
+    class ClusterInfo
+      using Refinements::HashExtensions
+
+      attr_reader :active_nodes, :total_nodes, :active_processors, :total_processors, :active_gpu_nodes,
+                  :total_gpu_nodes, :active_gpus, :total_gpus
+
+      def initialize(opts = {})
+        opts = opts.transform_keys(&:to_sym)
+        @active_nodes        = opts.fetch(:active_nodes, nil).to_i
+        @total_nodes         = opts.fetch(:total_nodes, nil).to_i
+        @active_processors   = opts.fetch(:active_processors, nil).to_i
+        @total_processors    = opts.fetch(:total_processors, nil).to_i
+        @active_gpus         = opts.fetch(:active_gpus, nil).to_i
+        @total_gpus          = opts.fetch(:total_gpus, nil).to_i
+      end
+
+      def to_h
+        {
+          active_nodes: active_nodes,
+          total_nodes: total_nodes,
+          active_processors: active_processors,
+          total_processors: total_processors,
+          active_gpus: active_gpus,
+          total_gpus: total_gpus
+        }
+      end
+    end
+  end
+end

data/lib/ood_core/job/info.rb

@@ -65,6 +65,10 @@ module OodCore
       # @return [Object] native info
       attr_reader :native
 
+      # Number of gpus allocated for job
+      # @return [Integer, nil] allocated total number of gpus
+      attr_reader :gpus
+
       # List of job array child task statuses
       # @note only relevant for job arrays
       # @return [Array<Task>] tasks
@@ -86,15 +90,16 @@ module OodCore
       # @param dispatch_time [#to_i, nil] dispatch time
       # @param tasks [Array<Hash>] tasks e.g. { id: '12345.owens-batch', status: :running }
       # @param native [Object] native info
+      # @param gpus [#to_i, 0] allocated total number of gpus
       def initialize(id:, status:, allocated_nodes: [], submit_host: nil,
                      job_name: nil, job_owner: nil, accounting_id: nil,
                      procs: nil, queue_name: nil, wallclock_time: nil,
                      wallclock_limit: nil, cpu_time: nil, submission_time: nil,
-                     dispatch_time: nil, native: nil, tasks: [],
+                     dispatch_time: nil, native: nil, gpus: 0, tasks: [],
                      **_)
         @id              = id.to_s
         @status          = Status.new(state: status.to_sym)
-        @allocated_nodes = allocated_nodes.map { |n| NodeInfo.new(n.to_h) }
+        @allocated_nodes = allocated_nodes.map { |n| NodeInfo.new(**n.to_h) }
         @submit_host     = submit_host     && submit_host.to_s
         @job_name        = job_name        && job_name.to_s
         @job_owner       = job_owner       && job_owner.to_s
@@ -111,6 +116,7 @@ module OodCore
         @status = job_array_aggregate_status unless @tasks.empty?
 
         @native          = native
+        @gpus            = gpus            && gpus.to_i
       end
 
       # Create a new Info for a child task
@@ -147,10 +153,15 @@ module OodCore
           submission_time: submission_time,
           dispatch_time:   dispatch_time,
           native:          native,
+          gpus:            gpus,
           tasks: tasks
         }
       end
 
+      def gpu?
+        gpus.positive?
+      end
+
       # The comparison operator
       # @param other [#to_h] object to compare against
       # @return [Boolean] whether objects are equivalent

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.19.0"
+  VERSION = "0.20.2"
 end

data/lib/ood_core.rb

@@ -11,6 +11,7 @@ module OodCore
     require "ood_core/job/node_info"
     require "ood_core/job/script"
     require "ood_core/job/info"
+    require "ood_core/job/cluster_info"
     require "ood_core/job/status"
     require "ood_core/job/adapter"
     require "ood_core/job/factory"

data/ood_core.gemspec

@@ -20,14 +20,15 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.2.0"
+  spec.required_ruby_version = ">= 2.5.0"
 
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
   spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
+  spec.add_runtime_dependency "rexml", "~> 3.2"
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0.1"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"
   spec.add_development_dependency "timecop", "~> 0.8"
-  spec.add_development_dependency "climate_control", "~> 1.0.0"
+  spec.add_development_dependency "climate_control", "~> 1.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.20.2
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-02 00:00:00.000000000 Z
+date: 2022-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -46,6 +46,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.6
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,14 +136,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.1
 description: Open OnDemand core library that provides support for an HPC Center to
   globally define HPC services that web applications can then take advantage of.
 email:
@@ -196,6 +210,7 @@ files:
 - lib/ood_core/job/adapters/torque/error.rb
 - lib/ood_core/job/adapters/torque/ffi.rb
 - lib/ood_core/job/array_ids.rb
+- lib/ood_core/job/cluster_info.rb
 - lib/ood_core/job/factory.rb
 - lib/ood_core/job/info.rb
 - lib/ood_core/job/node_info.rb
@@ -219,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="