ood_core 0.18.1 → 0.20.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a110306c66de3f349e7a5569cedc3ea02dfedd5dfaa360b352f8689af113f98b
-  data.tar.gz: ae608343e63bb98e6383fea71af53943d7136bdf72d8bf46b653f0c19801fcec
+  metadata.gz: 0ad31cf26a48b0b388b5c4f1bc83583515837a82c1a9419e51f724d135c4820c
+  data.tar.gz: 9eccd2b14920adf24f4273abd9355b4061a6a082913011757c3d636ba9d7d485
 SHA512:
-  metadata.gz: 8a6b9928561a6dba1b84cbb2ac58d389b84e8317589648c483382c166c81982859fb74f68f76297a25319faed06712c6256abdf1c6a5e0175be939aa0392f283
-  data.tar.gz: 21396c77e39329f9d7b6112c7900dd7ffa51d695b137d15089c487799ed16e3f74aea1f1dfab9958e2928fb98f49db098f865906c53abf667a8ed64ceda5dc53
+  metadata.gz: 126c139985ca62fdfca217ee8ea2fade4292d2f8aac110d51ad94edda36aff6e8c28fbe1876981010dc2ac395eb5cebea0439add3cafbb844a023ff699b4d841
+  data.tar.gz: 6d17cc409aff1b7d7098b409451c93e0660a855c4b4b36f8a49afea5cc306aeff5181e76305c15b056a5234e401f613bc428a3e49d98e292c3b584aaf4394a9a

data/CHANGELOG.md

@@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## [0.20.1] - 07-21-2022
+
+- Fixed turbovnc compatability issue with the -nohttpd flag in [767](https://github.com/OSC/ood_core/pull/767).
+
+## [0.20.0] - 06-03-2022
+
+- Adapters can now respond to `cluster_info` in [752](https://github.com/OSC/ood_core/pull/752). This returns information about the cluster like how many nodes are available and so on. Only Slurm support in this release.
+-  `OodCore::Job::Info` now has a `gpus` attribute in [753](https://github.com/OSC/ood_core/pull/753). Only Slurm support in this release.
+- Support Ruby 3 in [759](https://github.com/OSC/ood_core/pull/759)
+
+## [0.19.0] - 02-03-2022
+
+### Added
+
+- Systemd adapter in [743](https://github.com/OSC/ood_core/pull/743).
+
+### Fixed
+
+- The linux host adapter is a little more portable in [333](https://github.com/OSC/ood_core/pull/333).
+- Improved pod security for the k8s adapter in [748](https://github.com/OSC/ood_core/pull/748).
+
 ## [0.18.1] - 10-18-2021
 
 ### Fixed
@@ -411,7 +432,10 @@ Functionally the same as [0.17.3] but with some CI updates.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.18.1...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.20.1...HEAD
+[0.20.1]: https://github.com/OSC/ood_core/compare/v0.20.0...v0.20.1
+[0.20.0]: https://github.com/OSC/ood_core/compare/v0.19.0...v0.20.0
+[0.19.0]: https://github.com/OSC/ood_core/compare/v0.18.1...v0.19.0
 [0.18.1]: https://github.com/OSC/ood_core/compare/v0.18.0...v0.18.1
 [0.18.0]: https://github.com/OSC/ood_core/compare/v0.17.8...v0.18.0
 [0.17.6]: https://github.com/OSC/ood_core/compare/v0.17.5...v0.17.6

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2018 Ohio Supercomputer Center
+Copyright (c) 2017-2022 Ohio Supercomputer Center
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/lib/ood_core/batch_connect/templates/vnc.rb

@@ -86,8 +86,13 @@ module OodCore
                 # Clean up any old VNC sessions that weren't cleaned before
                 #{vnc_clean}
 
+                # for turbovnc 3.0 compatability.
+                if timeout 2 vncserver --help 2>&1 | grep 'nohttpd' >/dev/null 2>&1; then
+                  HTTPD_OPT='-nohttpd'
+                fi
+
                 # Attempt to start VNC server
-                VNC_OUT=$(vncserver -log "#{vnc_log}" -rfbauth "#{vnc_passwd}" -nohttpd -noxstartup #{vnc_args} 2>&1)
+                VNC_OUT=$(vncserver -log "#{vnc_log}" -rfbauth "#{vnc_passwd}" $HTTPD_OPT -noxstartup #{vnc_args} 2>&1)
                 VNC_PID=$(pgrep -s 0 Xvnc) # the script above will daemonize the Xvnc process
                 echo "${VNC_OUT}"
 

data/lib/ood_core/job/adapter.rb

@@ -33,6 +33,15 @@ module OodCore
         raise NotImplementedError, "subclass did not define #submit"
       end
 
+      # Retrieve the number of active and total cpus, nodes, and gpus
+      # @abstract Subclass is expected to implement {#cluster_stats}
+      # @raise [NotImplementedError] if subclass did not define {#cluster_stats}
+      # @return [ClusterInfo] Object containing quantified statistics about the
+      #   cluster's active/total cpus, nodes, and gpus
+      def cluster_info
+        raise NotImplementedError, "subclass did not define #cluster_stats"
+      end
+
       # Retrieve info for all jobs from the resource manager
       # @abstract Subclass is expected to implement {#info_all}
       # @raise [NotImplementedError] if subclass did not define {#info_all}

data/lib/ood_core/job/adapters/ccq.rb

@@ -228,7 +228,7 @@ module OodCore
           data_hash[:submission_time] = raw['dateSubmitted'].to_i
           data_hash[:queue_name] = raw['criteriaPriority']
 
-          Info.new(data_hash)
+          Info.new(**data_hash)
         end
 
         # extended data is just lines of 'key: value' value, so parse
@@ -242,7 +242,7 @@ module OodCore
 
           data.to_s.lines.drop(1).each do |line|
             match_data = ccqstat_regex.match(line)
-            infos << Info.new(ccqstat_match_to_hash(match_data)) if valid_ccqstat_match?(match_data)
+            infos << Info.new(**ccqstat_match_to_hash(match_data)) if valid_ccqstat_match?(match_data)
           end
 
           infos

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -93,7 +93,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
 
   def info(id)
     pod_json = safe_call('get', 'pod', id)
-    return OodCore::Job::Info.new({ id: id, status: 'completed' }) if pod_json.empty?
+    return OodCore::Job::Info.new(**{ id: id, status: 'completed' }) if pod_json.empty?
 
     service_json = safe_call('get', 'service', service_name(id))
     secret_json = safe_call('get', 'secret', secret_name(id))

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -31,7 +31,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
 
     pod_hash.deep_merge!(service_hash)
     pod_hash.deep_merge!(secret_hash)
-    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(pod_hash)
+    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(**pod_hash)
   rescue NoMethodError
     raise K8sDataError, "unable to read data correctly from json"
   end

data/lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb

@@ -2,8 +2,8 @@
 class OodCore::Job::Adapters::Kubernetes::K8sJobInfo < OodCore::Job::Info
   attr_reader :ood_connection_info
 
-  def initialize(ood_connection_info: {}, **options)
-    super(options)
-    @ood_connection_info = ood_connection_info
+  def initialize(options)
+    super(**options)
+    @ood_connection_info = options[:ood_connection_info]
   end
 end

data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb

@@ -106,7 +106,9 @@ spec:
       allowPrivilegeEscalation: false
       capabilities:
         drop:
-        - all
+        - ALL
+      seccompProfile:
+        type: RuntimeDefault
       privileged: false
   <%- unless spec.init_containers.empty? -%>
   initContainers:
@@ -152,7 +154,9 @@ spec:
       allowPrivilegeEscalation: false
       capabilities:
         drop:
-        - all
+        - ALL
+      seccompProfile:
+        type: RuntimeDefault
       privileged: false
   <%- end # init container loop -%>
   <%- end # if init containers -%>

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -73,7 +73,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     # Get the tmux pane PID for the target session
     pane_pid=$(tmux list-panes -aF '\#{session_name} \#{pane_pid}' | grep '#{session_name}' | cut -f 2 -d ' ')
     # Find the Singularity sinit PID child of the pane process
-    pane_sinit_pid=$(pstree -p -l "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
+    pane_sinit_pid=$(pstree -p -l "$pane_pid" | egrep -o 'sinit[(][[:digit:]]*|shim-init[(][[:digit:]]*' | grep -o '[[:digit:]]*') 
     # Kill sinit which stops both Singularity-based processes and the tmux session
     kill "$pane_sinit_pid"
     SCRIPT

data/lib/ood_core/job/adapters/lsf.rb

@@ -16,7 +16,7 @@ module OodCore
       # @option config [#to_h] :bin_overrides ({}) Optional overrides to LSF client executables
       # @option config [#to_s] :submit_host ('') Host to submit commands to
       def self.build_lsf(config)
-        batch = Adapters::Lsf::Batch.new(config.to_h.symbolize_keys)
+        batch = Adapters::Lsf::Batch.new(**config.to_h.symbolize_keys)
         Adapters::Lsf.new(batch: batch)
       end
     end

data/lib/ood_core/job/adapters/slurm.rb

@@ -36,6 +36,13 @@ module OodCore
         using Refinements::HashExtensions
         using Refinements::ArrayExtensions
 
+        # Get integer representing the number of gpus used by a node or job,
+        # calculated from gres string
+        # @return [Integer] the number of gpus in gres
+        def gpus_from_gres(gres)
+          gres.to_s.scan(/gpu:[^,]*(\d+)/).flatten.map(&:to_i).sum
+        end
+
         # Object used for simplified communication with a Slurm batch server
         # @api private
         class Batch
@@ -98,6 +105,22 @@ module OodCore
             @strict_host_checking = strict_host_checking
           end
 
+          # Get a ClusterInfo object containing information about the given cluster
+          # @return [ClusterInfo] object containing cluster details
+          def get_cluster_info
+            node_cpu_info = call("sinfo", "-aho %A/%D/%C").strip.split('/')
+            gres_length = call("sinfo", "-o %G").lines.map(&:strip).map(&:length).max + 2
+            gres_lines = call("sinfo", "-ahNO ,nodehost,gres:#{gres_length},gresused:#{gres_length}")
+                         .lines.uniq.map(&:split)
+            ClusterInfo.new(active_nodes: node_cpu_info[0].to_i,
+                            total_nodes: node_cpu_info[2].to_i,
+                            active_processors: node_cpu_info[3].to_i,
+                            total_processors: node_cpu_info[6].to_i,
+                            active_gpus: gres_lines.sum { |line| gpus_from_gres(line[2]) },
+                            total_gpus: gres_lines.sum { |line| gpus_from_gres(line[1]) }
+            )
+          end
+
           # Get a list of hashes detailing each of the jobs on the batch server
           # @example Status info for all jobs
           #   my_batch.get_jobs
@@ -454,6 +477,12 @@ module OodCore
           raise JobAdapterError, e.message
         end
 
+        # Retrieve info about active and total cpus, gpus, and nodes
+        # @return [Hash] information about cluster usage
+        def cluster_info
+          @slurm.get_cluster_info
+        end
+
         # Retrieve info for all jobs from the resource manager
         # @raise [JobAdapterError] if something goes wrong getting job info
         # @return [Array<Info>] information describing submitted jobs
@@ -643,7 +672,8 @@ module OodCore
               cpu_time: nil,
               submission_time: v[:submit_time] ? Time.parse(v[:submit_time]) : nil,
               dispatch_time: (v[:start_time].nil? || v[:start_time] == "N/A") ? nil : Time.parse(v[:start_time]),
-              native: v
+              native: v,
+              gpus: gpus_from_gres(v[:gres])
             )
           end
 

data/lib/ood_core/job/adapters/systemd/launcher.rb

@@ -0,0 +1,252 @@
+require 'erb'
+require 'etc'
+require 'pathname'
+require 'securerandom'
+require 'shellwords'
+require 'time'
+
+# Object used for simplified communication SSH hosts
+#
+# @api private
+class OodCore::Job::Adapters::LinuxSystemd::Launcher
+  attr_reader :debug, :site_timeout, :session_name_label, :ssh_hosts,
+    :strict_host_checking, :username
+  # The root exception class that all LinuxSystemd adapter-specific exceptions inherit
+  # from
+  class Error < StandardError; end
+
+  # @param debug Whether the adapter should be used in debug mode
+  # @param site_timeout [#to_i] A period after which the job should be killed or nil
+  # @param ssh_hosts List of hosts to check when scanning for running jobs
+  # @param strict_host_checking Allow SSH to perform strict host checking
+  # @param submit_host The SSH-able host
+  def initialize(
+    debug: false,
+    site_timeout: nil,
+    ssh_hosts:,
+    strict_host_checking: false,
+    submit_host:,
+    **_
+  )
+    @debug = !! debug
+    @site_timeout = site_timeout.to_i
+    @session_name_label = 'ondemand'
+    @ssh_hosts = ssh_hosts
+    @strict_host_checking = strict_host_checking
+    @submit_host = submit_host
+    @username = Etc.getlogin
+  end
+
+  # @param hostname [#to_s] The hostname to submit the work to
+  # @param script [OodCore::Job::Script] The script object defining the work
+  def start_remote_session(script)
+    cmd = ssh_cmd(submit_host(script), ['/usr/bin/env', 'bash'])
+
+    session_name = unique_session_name
+    output = call(*cmd, stdin: wrapped_script(script, session_name))
+    hostname = parse_hostname(output)
+
+    "#{session_name}@#{hostname}"
+  end
+
+  def stop_remote_session(session_name, hostname)
+    cmd = ssh_cmd(hostname, ['/usr/bin/env', 'bash'])
+
+    kill_cmd = <<~SCRIPT
+    # stop the session by name
+    systemctl --user stop #{session_name}.service
+    SCRIPT
+
+    call(*cmd, stdin: kill_cmd)
+  rescue Error => e
+    interpret_and_raise(e)
+  end
+
+  def list_remote_sessions(host: nil)
+    host_list = (host) ? [host] : ssh_hosts
+
+    host_list.map {
+      |hostname| list_remote_systemd_session(hostname)
+    }.flatten.sort_by {
+      |hsh| hsh[:session_name]
+    }
+  end
+
+  def submit_host(script = nil)
+    if script && script.native && script.native['submit_host_override']
+      script.native['submit_host_override']
+    else
+      @submit_host
+    end
+  end
+
+  private
+
+  # Call a forked Slurm command for a given cluster
+  def call(cmd, *args, env: {}, stdin: "")
+    args  = args.map(&:to_s)
+    env = env.to_h
+    o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
+    s.success? ? o : raise(Error, e)
+  end
+
+  # The full command to ssh into the destination host and execute the command.
+  # SSH options include:
+  # -t Force pseudo-terminal allocation (required to allow tmux to run)
+  # -o BatchMode=yes (set mode to be non-interactive)
+  # if ! strict_host_checking
+  # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
+  # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
+  #
+  # @param destination_host [#to_s] the destination host you wish to ssh into
+  # @param cmd [Array<#to_s>] the command to be executed on the destination host
+  def ssh_cmd(destination_host, cmd)
+    if strict_host_checking
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        "#{username}@#{destination_host}"
+      ].concat(cmd)
+    else
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        '-o', 'StrictHostKeyChecking=no',
+        "#{username}@#{destination_host}"
+      ].concat(cmd)
+    end
+  end
+
+  def shell
+    ENV['SHELL'] || '/bin/bash'
+  end
+
+  # Wraps a user-provided script into a systemd-run transient service
+  def wrapped_script(script, session_name)
+    content = script.content
+    unless user_script_has_shebang?(script)
+     content = "#!#{shell}\n#{content}"
+    end
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/script_wrapper.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'arguments' => script_arguments(script),
+        'cd_to_workdir' => (script.workdir) ? "cd #{script.workdir}" : '',
+        'debug' => debug,
+        'email_on_terminated' => script_email_on_event(script, 'terminated'),
+        'email_on_start' => script_email_on_event(script, 'started'),
+        'environment' => export_env(script),
+        'error_path' => error_path(script),
+        'job_name' => script.job_name.to_s,
+        'output_path' => (script.output_path) ? script.output_path.to_s : '/dev/null',
+        'script_content' => content,
+        'script_timeout' => script_timeout(script),
+        'session_name' => session_name,
+        'ssh_hosts' => ssh_hosts,
+        'workdir' => (script.workdir) ? script.workdir.to_s : '/tmp',
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  # Generate the environment export block for this script
+  def export_env(script)
+    environment = script.job_environment
+    (environment ? environment : {}).map{
+      |key, value| "export #{key}=#{Shellwords.escape(value)}"
+    }.sort.join("\n")
+  end
+
+  def script_timeout(script)
+    wall_time = script.wall_time.to_i
+    if wall_time == 0
+      # this is the only way it can be 0
+      # so make it into infinify for systemd to never terminate
+      site_timeout == 0 ? 'infinity' : site_timeout
+    elsif site_timeout != 0
+      [wall_time, site_timeout].min
+    else
+      wall_time
+    end
+  end
+
+  def script_arguments(script)
+    return '' unless script.args
+
+    Shellwords.join(script.args)
+  end
+
+  def script_email_on_event(script, event)
+    return false unless script.email && script.send("email_on_#{event}")
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/email.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'email_recipients' => script.email.map{|addr| Shellwords.escape(addr)}.join(', '),
+        'job_name' => (script.job_name) ? script.job_name : 'LinuxHost_Adapter_Job',
+        'job_status' => event
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  def unique_session_name
+    "#{session_name_label}-#{SecureRandom.alphanumeric(10)}"
+  end
+
+  # List all Systemd sessions on destination_host started by this adapter
+  def list_remote_systemd_session(destination_host)
+    cmd = ssh_cmd(destination_host, ['systemctl', '--user', 'show', '-t', 'service', '--state=running', "#{session_name_label}-*"])
+
+    # individual units are separated with an empty line
+    call(*cmd).split("\n\n").map do |oneunit|
+      Hash[oneunit.split("\n").map{ |line| line.split('=',2) }].tap do |session_hash|
+         session_hash[:session_name] = session_hash['Id'].delete_suffix('.service')
+         session_hash[:destination_host] = destination_host
+         session_hash[:id] = "#{session_hash[:session_name]}@#{destination_host}"
+         session_hash[:session_created] = Time.parse(session_hash['ExecMainStartTimestamp'])
+	 session_hash[:job_name] = session_hash['Description']
+      end
+    end
+  rescue Error => e
+    interpret_and_raise(e)
+    []
+  end
+
+  def user_script_has_shebang?(script)
+    return false if script.content.empty?
+    script.content.split("\n").first.start_with?('#!/')
+  end
+
+  def error_path(script)
+    return script.error_path.to_s if script.error_path
+    return script.output_path.to_s if script.output_path
+
+    '/dev/null'
+  end
+
+  # under some conditions tmux returns status code 1 but it's not an actual
+  # error. These are when the session is not found or there are no sessions
+  # at all.
+  def interpret_and_raise(error)
+    if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
+      nil
+    elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
+      nil
+    else
+      raise error
+    end
+  end
+
+  def parse_hostname(output)
+    output.split($/).map do |line|
+      line[/^HOSTNAME:(.*)$/, 1]
+    end.compact.last.to_s
+  end
+end

data/lib/ood_core/job/adapters/systemd/templates/email.erb.sh

@@ -0,0 +1,9 @@
+if command -v mail; then
+cat << EMAIL_CONTENT | mail -s "Job <%= job_name %> has <%= job_status %>" <%= email_recipients %>
+Greetings,
+
+Your job <%= job_name %> has <%= job_status %>.
+
+- The OnDemand Linux Systemd Adapter
+EMAIL_CONTENT
+fi

data/lib/ood_core/job/adapters/systemd/templates/script_wrapper.erb.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+SSH_HOSTS=(<%= ssh_hosts.join(' ').to_s %>)
+hostnames=`hostname -A`
+for host in ${SSH_HOSTS[@]}
+do
+    if [[ " ${hostnames[@]} " =~ " ${host} " ]]; then
+        hostname=$host
+    fi
+done
+
+if [ -z "$hostname" ]; then
+    printf >&2 "ERROR: Can't start job on [${hostnames[@]}] because it does not match any hostname configured \nin ssh_hosts [${SSH_HOSTS[@]}]. The output of 'hostname -A' must match an entry in ssh_hosts \nfrom the cluster configuration."
+    exit 1
+fi
+
+echo ""
+echo "HOSTNAME:$hostname"
+
+# we need this user to be enabled for lingering or else the newly started
+# service will end as soon as the ssh session starting has exited
+loginctl enable-linger
+
+# Put the script into a temp file on localhost
+systemd_service_file="<%= workdir %>/systemd_service.sh"
+systemd_service_file_pre="<%= workdir %>/systemd_pre.sh"
+systemd_service_file_post="<%= workdir %>/systemd_post.sh"
+
+cat << 'SYSTEMD_EXEC_PRE' > "$systemd_service_file_pre"
+#!/bin/bash
+<%= cd_to_workdir %>
+<% if email_on_start %>
+<%= email_on_start %>
+<% end %>
+SYSTEMD_EXEC_PRE
+
+cat << 'SYSTEMD_EXEC_POST' > "$systemd_service_file_post"
+#!/bin/bash
+<%= cd_to_workdir %>
+<% if email_on_terminated %>
+<%= email_on_terminated %>
+<% end %>
+SYSTEMD_EXEC_POST
+
+# Create an executable for systemd service to run
+# Escaped HEREDOC means that we do not have to worry about Shell.escape-ing script_content
+cat << 'SYSTEMD_EXEC' > "$systemd_service_file"
+<%= script_content %>
+SYSTEMD_EXEC
+
+# Run the script inside a transient systemd user service
+chmod +x "$systemd_service_file_pre" "$systemd_service_file" "$systemd_service_file_post"
+<%= cd_to_workdir %>
+systemd-run --user -r --no-block --unit=<%= session_name %> -p RuntimeMaxSec=<%= script_timeout %> \
+	-p ExecStartPre="$systemd_service_file_pre" -p ExecStartPost="$systemd_service_file_post" \
+	-p StandardOutput="file:<%= output_path %>" -p StandardError="file:<%= error_path %>" \
+	-p Description="<%= job_name %>" "$systemd_service_file"

data/lib/ood_core/job/adapters/systemd.rb

@@ -0,0 +1,230 @@
+require "ood_core/refinements/hash_extensions"
+require "ood_core/refinements/array_extensions"
+require "ood_core/job/adapters/helper"
+require "set"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      # Build the LinuxSystemd adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :debug (false) Use the adapter in a debug mode
+      # @option config [Object] :max_timeout (nil) The longest 'wall_clock' permissible
+      # @option config [Object] :ssh_hosts (nil) The list of permissable hosts, defaults to :submit_host
+      # @option config [Object] :strict_host_checking (true) Set to false to disable strict host checking and updating the known_hosts file
+      # @option config [Object] :submit_host The SSH target to connect to, may be the head of a round-robin
+      def self.build_systemd(config)
+        c = config.to_h.symbolize_keys
+        debug = c.fetch(:debug, false)
+        max_timeout = c.fetch(:max_timeout, nil)
+        ssh_hosts = c.fetch(:ssh_hosts, [c[:submit_host]])
+        strict_host_checking = c.fetch(:strict_host_checking, true)
+        submit_host = c[:submit_host]
+
+        Adapters::LinuxSystemd.new(
+          ssh_hosts: ssh_hosts,
+          launcher: Adapters::LinuxSystemd::Launcher.new(
+            debug: debug,
+            max_timeout: max_timeout,
+            ssh_hosts: ssh_hosts,
+            strict_host_checking: strict_host_checking,
+            submit_host: submit_host,
+          )
+        )
+      end
+    end
+
+    module Adapters
+      # An adapter object that describes the communication with a remote host
+      # for job management.
+      class LinuxSystemd < Adapter
+        using Refinements::ArrayExtensions
+
+        require "ood_core/job/adapters/systemd/launcher"
+
+        def initialize(ssh_hosts:, launcher:)
+          @launcher = launcher
+          @ssh_hosts = Set.new(ssh_hosts)
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @param script [Script] script object that describes the script and
+        #   attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afterok [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afternotok [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afterany [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @raise [JobAdapterError] if something goes wrong submitting a job
+        # @return [String] the job id returned after successfully submitting a
+        #   job
+        # @see Adapter#submit
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          unless (after.empty? && afterok.empty? && afternotok.empty? && afterany.empty?)
+            raise JobAdapterError, 'Scheduling subsequent jobs is not available.'
+          end
+
+          @launcher.start_remote_session(script)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs from the resource manager
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Array<Info>] information describing submitted jobs
+        # @see Adapter#info_all
+        def info_all(attrs: nil, host: nil)
+          host_permitted?(host) if host
+
+          @launcher.list_remote_sessions(host: host).map{
+            |ls_output| ls_to_info(ls_output)
+          }
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs for a given owner or owners from the
+        # resource manager
+        # Note: owner and attrs are present only to complete the interface and are ignored
+        # Note: since this API is used in production no errors or warnings are thrown / issued
+        # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Array<Info>] information describing submitted jobs
+        def info_where_owner(_, attrs: nil)
+          info_all
+        end
+
+        # Iterate over each job Info object
+        # @param attrs [Array<symbol>] attrs is present only to complete the interface and is ignored
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_all_each(attrs: nil)
+          return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+          info_all(attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Iterate over each job Info object
+        # @param owner [#to_s, Array<#to_s>] owner is present only to complete the interface and is ignored
+        # @param attrs [Array<symbol>] attrs is present only to complete the interface and is ignored
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_where_owner_each(owner, attrs: nil)
+          return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+          info_where_owner(owner, attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Whether the adapter supports job arrays
+        # @return [Boolean] - false
+        def supports_job_arrays?
+          false
+        end
+
+        # Retrieve job info from the SSH host
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Info] information describing submitted job
+        # @see Adapter#info
+        def info(id)
+          _, host = parse_job_id(id)
+          job = info_all(host: host).select{|info| info.id == id}.first
+          (job) ? job : Info.new(id: id, status: :completed)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve job status from resource manager
+        # @note Optimized slightly over retrieving complete job information from server
+        # @abstract Subclass is expected to implement {#status}
+        # @raise [NotImplementedError] if subclass did not define {#status}
+        # @param id [#to_s] the id of the job
+        # @return [Status] status of job
+        def status(id)
+          _, host = parse_job_id(id)
+          job = info_all(host: host).select{|info| info.id == id}.first
+
+          Status.new(state: (job) ? :running : :completed)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Put the submitted job on hold
+        # @abstract Subclass is expected to implement {#hold}
+        # @raise [NotImplementedError] if subclass did not define {#hold}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def hold(id)
+          # Consider sending SIGSTOP?
+          raise NotImplementedError, "subclass did not define #hold"
+        end
+
+        # Release the job that is on hold
+        # @abstract Subclass is expected to implement {#release}
+        # @raise [NotImplementedError] if subclass did not define {#release}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def release(id)
+          # Consider sending SIGCONT
+          raise NotImplementedError, "subclass did not define #release"
+        end
+
+        # Delete the submitted job
+        # @abstract Subclass is expected to implement {#delete}
+        # @raise [NotImplementedError] if subclass did not define {#delete}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def delete(id)
+          session_name, destination_host = parse_job_id(id)
+          @launcher.stop_remote_session(session_name, destination_host)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        def directive_prefix
+          nil
+        end
+
+        private
+
+        def host_permitted?(destination_host)
+          raise JobAdapterError, "Requested destination host (#{destination_host}) not permitted" unless @ssh_hosts.include?(destination_host)
+        end
+
+        def parse_job_id(id)
+          raise JobAdapterError, "#{id} is not a valid LinuxSystemd adapter id because it is missing the '@'." unless id.include?('@')
+
+          return id.split('@')
+        end
+
+        # Convert the returned Hash into an Info object
+        def ls_to_info(ls_output)
+          started = ls_output[:session_created].to_i
+          now = Time.now.to_i
+          ellapsed = now - started
+          Info.new(
+              accounting_id: nil,
+              allocated_nodes: [NodeInfo.new(name: ls_output[:destination_host], procs: 1)],
+              cpu_time: ellapsed,
+              dispatch_time: started,
+              id: ls_output[:id],
+              job_name: ls_output[:job_name],
+              job_owner: Etc.getlogin,
+              native: ls_output,
+              procs: 1,
+              queue_name: "LinuxSystemd adapter for #{@submit_host}",
+              status: :running,
+              submission_time: ellapsed,
+              submit_host: @submit_host,
+              wallclock_time: ellapsed
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/job/cluster_info.rb

@@ -0,0 +1,32 @@
+module OodCore
+  module Job
+    # An object that contains details about the cluster's active and total nodes, processors, and gpus
+    class ClusterInfo
+      using Refinements::HashExtensions
+
+      attr_reader :active_nodes, :total_nodes, :active_processors, :total_processors, :active_gpu_nodes,
+                  :total_gpu_nodes, :active_gpus, :total_gpus
+
+      def initialize(opts = {})
+        opts = opts.transform_keys(&:to_sym)
+        @active_nodes        = opts.fetch(:active_nodes, nil).to_i
+        @total_nodes         = opts.fetch(:total_nodes, nil).to_i
+        @active_processors   = opts.fetch(:active_processors, nil).to_i
+        @total_processors    = opts.fetch(:total_processors, nil).to_i
+        @active_gpus         = opts.fetch(:active_gpus, nil).to_i
+        @total_gpus          = opts.fetch(:total_gpus, nil).to_i
+      end
+
+      def to_h
+        {
+          active_nodes: active_nodes,
+          total_nodes: total_nodes,
+          active_processors: active_processors,
+          total_processors: total_processors,
+          active_gpus: active_gpus,
+          total_gpus: total_gpus
+        }
+      end
+    end
+  end
+end

data/lib/ood_core/job/info.rb

@@ -65,6 +65,10 @@ module OodCore
       # @return [Object] native info
       attr_reader :native
 
+      # Number of gpus allocated for job
+      # @return [Integer, nil] allocated total number of gpus
+      attr_reader :gpus
+
       # List of job array child task statuses
       # @note only relevant for job arrays
       # @return [Array<Task>] tasks
@@ -86,15 +90,16 @@ module OodCore
       # @param dispatch_time [#to_i, nil] dispatch time
       # @param tasks [Array<Hash>] tasks e.g. { id: '12345.owens-batch', status: :running }
       # @param native [Object] native info
+      # @param gpus [#to_i, 0] allocated total number of gpus
       def initialize(id:, status:, allocated_nodes: [], submit_host: nil,
                      job_name: nil, job_owner: nil, accounting_id: nil,
                      procs: nil, queue_name: nil, wallclock_time: nil,
                      wallclock_limit: nil, cpu_time: nil, submission_time: nil,
-                     dispatch_time: nil, native: nil, tasks: [],
+                     dispatch_time: nil, native: nil, gpus: 0, tasks: [],
                      **_)
         @id              = id.to_s
         @status          = Status.new(state: status.to_sym)
-        @allocated_nodes = allocated_nodes.map { |n| NodeInfo.new(n.to_h) }
+        @allocated_nodes = allocated_nodes.map { |n| NodeInfo.new(**n.to_h) }
         @submit_host     = submit_host     && submit_host.to_s
         @job_name        = job_name        && job_name.to_s
         @job_owner       = job_owner       && job_owner.to_s
@@ -111,6 +116,7 @@ module OodCore
         @status = job_array_aggregate_status unless @tasks.empty?
 
         @native          = native
+        @gpus            = gpus            && gpus.to_i
       end
 
       # Create a new Info for a child task
@@ -147,10 +153,15 @@ module OodCore
           submission_time: submission_time,
           dispatch_time:   dispatch_time,
           native:          native,
+          gpus:            gpus,
           tasks: tasks
         }
       end
 
+      def gpu?
+        gpus.positive?
+      end
+
       # The comparison operator
       # @param other [#to_h] object to compare against
       # @return [Boolean] whether objects are equivalent

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.18.1"
+  VERSION = "0.20.1"
 end

data/lib/ood_core.rb

@@ -11,6 +11,7 @@ module OodCore
     require "ood_core/job/node_info"
     require "ood_core/job/script"
     require "ood_core/job/info"
+    require "ood_core/job/cluster_info"
     require "ood_core/job/status"
     require "ood_core/job/adapter"
     require "ood_core/job/factory"

data/ood_core.gemspec

@@ -20,14 +20,15 @@ Gem::Specification.new do |spec|
   spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.2.0"
+  spec.required_ruby_version = ">= 2.7.0"
 
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
   spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
+  spec.add_runtime_dependency "rexml", "~> 3.2"
   spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "rake", "~> 13.0.1"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"
   spec.add_development_dependency "timecop", "~> 0.8"
-  spec.add_development_dependency "climate_control", "~> 0.2.0"
+  spec.add_development_dependency "climate_control", "~> 1.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.18.1
+  version: 0.20.1
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-18 00:00:00.000000000 Z
+date: 2022-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -46,6 +46,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.9.6
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -122,14 +136,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.1.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.1.1
 description: Open OnDemand core library that provides support for an HPC Center to
   globally define HPC services that web applications can then take advantage of.
 email:
@@ -186,12 +200,17 @@ files:
 - lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb
 - lib/ood_core/job/adapters/sge/qstat_xml_r_listener.rb
 - lib/ood_core/job/adapters/slurm.rb
+- lib/ood_core/job/adapters/systemd.rb
+- lib/ood_core/job/adapters/systemd/launcher.rb
+- lib/ood_core/job/adapters/systemd/templates/email.erb.sh
+- lib/ood_core/job/adapters/systemd/templates/script_wrapper.erb.sh
 - lib/ood_core/job/adapters/torque.rb
 - lib/ood_core/job/adapters/torque/attributes.rb
 - lib/ood_core/job/adapters/torque/batch.rb
 - lib/ood_core/job/adapters/torque/error.rb
 - lib/ood_core/job/adapters/torque/ffi.rb
 - lib/ood_core/job/array_ids.rb
+- lib/ood_core/job/cluster_info.rb
 - lib/ood_core/job/factory.rb
 - lib/ood_core/job/info.rb
 - lib/ood_core/job/node_info.rb
@@ -215,7 +234,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="