ood_core 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3296708d7bc47f3379a9e4a6c845d3f25c5ccefb599f4b92406d9dffdaef220b
-  data.tar.gz: b6af9e90b67bc9a7a52203808d849d8800336b30b09bdb8ed204526d01bc92e9
+  metadata.gz: 52ba764b085dedb7eaeb06d95751f1804a50488e1859f980a7836d2d9032b95d
+  data.tar.gz: c2dc5edf395fe158960f33b80c554f3dc745f15e7ec1337b738683a0e1bbdc7f
 SHA512:
-  metadata.gz: 623ac6e6f8081d68a3e925d1150c9f20a0f613ccfb6837519d1b95d04533a72caa403c54327aad85dcea9c0694cc23941f40307d942623c095f53fed7fc32026
-  data.tar.gz: 0d785a9ade36b2f6f62f9ae55672091346aa4fb76bf358e6c00d4bc007623b8d1798813474665fc7b4d850d89e041fae5c2fefc9719fbe9f53a161a76127eaad
+  metadata.gz: 59915bae23a008a923c249d222e50548a7bee3438144068a29ae1cafdd489ca1229ee1a14f4f81e3fd065381f46f920bef24344fe633c7c578cb1f6a4f9a2a77
+  data.tar.gz: 8d2ca42c7f49158c8d321c21b79aff1c636df3c77bb7e71107db70371a34058d79c8a5ec32ca93883e7d3bcc7dc2202375144d23613f167ab089318d6270248c

data/CHANGELOG.md

@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.14.0] - 2020-10-01
+### Added 
+- Kubernetes adapter in PR [156](https://github.com/OSC/ood_core/pull/156)
+
+### Fixed
+- Catch Slurm times. [209](https://github.com/OSC/ood_core/pull/209)
+- LHA race condition in deleteing tmp files. [212](https://github.com/OSC/ood_core/pull/212)
+
 ## [0.13.0] - 2020-08-10
 ### Added
 - CloudyCluster CCQ Adapter
@@ -247,7 +255,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.13.0...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.14.0...HEAD
+[0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0
 [0.13.0]: https://github.com/OSC/ood_core/compare/v0.12.0...v0.13.0
 [0.12.0]: https://github.com/OSC/ood_core/compare/v0.11.4...v0.12.0
 [0.11.4]: https://github.com/OSC/ood_core/compare/v0.11.3...v0.11.4

data/README.md

@@ -6,7 +6,7 @@
 
 - Website: http://openondemand.org/
 - Website repo with JOSS publication: https://github.com/OSC/Open-OnDemand
-- Documentation: https://osc.github.io/ood-documentation/master/
+- Documentation: https://osc.github.io/ood-documentation/latest/
 - Main code repo: https://github.com/OSC/ondemand
 - Core library repo: https://github.com/OSC/ood_core
 

data/lib/ood_core/job/adapters/kubernetes.rb

@@ -0,0 +1,193 @@
+require "ood_core/refinements/hash_extensions"
+require "ood_core/refinements/array_extensions"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      def self.build_kubernetes(config)
+        batch = Adapters::Kubernetes::Batch.new(config.to_h.symbolize_keys, Adapters::Kubernetes::Helper.new)
+        Adapters::Kubernetes.new(batch)
+      end
+    end
+
+    module Adapters
+      class Kubernetes < Adapter
+
+        using Refinements::ArrayExtensions
+        using Refinements::HashExtensions
+
+        require "ood_core/job/adapters/kubernetes/batch"
+
+        attr_reader :batch
+
+        def initialize(batch)
+          @batch = batch
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @abstract Subclass is expected to implement {#submit}
+        # @raise [NotImplementedError] if subclass did not define {#submit}
+        # @example Submit job template to cluster
+        #   solver_id = job_adapter.submit(solver_script)
+        #   #=> "1234.server"
+        # @example Submit job that depends on previous job
+        #   post_id = job_adapter.submit(
+        #     post_script,
+        #     afterok: solver_id
+        #   )
+        #   #=> "1235.server"
+        # @param script [Script] script object that describes the
+        #   script and attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] this job may be scheduled for execution
+        #   at any point after dependent jobs have started execution
+        # @param afterok [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution only after dependent jobs have terminated with no errors
+        # @param afternotok [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution only after dependent jobs have terminated with errors
+        # @param afterany [#to_s, Array<#to_s>] this job may be scheduled for
+        #   execution after dependent jobs have terminated
+        # @return [String] the job id returned after successfully submitting a job
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          raise ArgumentError, 'Must specify the script' if script.nil?
+
+          batch.submit(script)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+
+        # Retrieve info for all jobs from the resource manager
+        # @abstract Subclass is expected to implement {#info_all}
+        # @raise [NotImplementedError] if subclass did not define {#info_all}
+        # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+        #   This array specifies only attrs you want, in addition to id and status.
+        #   If an array, the Info object that is returned to you is not guarenteed
+        #   to have a value for any attr besides the ones specified and id and status.
+        #
+        #   For certain adapters this may speed up the response since
+        #   adapters can get by without populating the entire Info object
+        # @return [Array<Info>] information describing submitted jobs
+        def info_all(attrs: nil)
+          batch.info_all(attrs: attrs)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs for a given owner or owners from the
+        # resource manager
+        # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+        # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+        #   This array specifies only attrs you want, in addition to id and status.
+        #   If an array, the Info object that is returned to you is not guarenteed
+        #   to have a value for any attr besides the ones specified and id and status.
+        #
+        #   For certain adapters this may speed up the response since
+        #   adapters can get by without populating the entire Info object
+        # @return [Array<Info>] information describing submitted jobs
+        def info_where_owner(owner, attrs: nil)
+          owner = Array.wrap(owner).map(&:to_s)
+
+          # must at least have job_owner to filter by job_owner
+          attrs = Array.wrap(attrs) | [:job_owner] unless attrs.nil?
+
+          info_all(attrs: attrs).select { |info| owner.include? info.job_owner }
+        end
+
+        # Iterate over each job Info object
+        # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+        #   This array specifies only attrs you want, in addition to id and status.
+        #   If an array, the Info object that is returned to you is not guarenteed
+        #   to have a value for any attr besides the ones specified and id and status.
+        #
+        #   For certain adapters this may speed up the response since
+        #   adapters can get by without populating the entire Info object
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_all_each(attrs: nil)
+          return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+          info_all(attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Iterate over each job Info object
+        # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+        # @param attrs [Array<symbol>] defaults to nil (and all attrs are provided) 
+        #   This array specifies only attrs you want, in addition to id and status.
+        #   If an array, the Info object that is returned to you is not guarenteed
+        #   to have a value for any attr besides the ones specified and id and status.
+        #
+        #   For certain adapters this may speed up the response since
+        #   adapters can get by without populating the entire Info object
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_where_owner_each(owner, attrs: nil)
+          return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+          info_where_owner(owner, attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Whether the adapter supports job arrays
+        # @return [Boolean] - assumes true; but can be overridden by adapters that
+        #   explicitly do not
+        def supports_job_arrays?
+          false
+        end
+
+        # Retrieve job info from the resource manager
+        # @abstract Subclass is expected to implement {#info}
+        # @raise [NotImplementedError] if subclass did not define {#info}
+        # @param id [#to_s] the id of the job
+        # @return [Info] information describing submitted job
+        def info(id)
+          batch.info(id.to_s)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve job status from resource manager
+        # @note Optimized slightly over retrieving complete job information from server
+        # @abstract Subclass is expected to implement {#status}
+        # @raise [NotImplementedError] if subclass did not define {#status}
+        # @param id [#to_s] the id of the job
+        # @return [Status] status of job
+        def status(id)
+          info(id).status
+        end
+
+        # Put the submitted job on hold
+        # @abstract Subclass is expected to implement {#hold}
+        # @raise [NotImplementedError] if subclass did not define {#hold}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def hold(id)
+          raise NotImplementedError, 'subclass did not define #hold'
+        end
+
+        # Release the job that is on hold
+        # @abstract Subclass is expected to implement {#release}
+        # @raise [NotImplementedError] if subclass did not define {#release}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def release(id)
+          raise NotImplementedError, 'subclass did not define #release'
+        end
+
+        # Delete the submitted job.
+        #
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def delete(id)
+          batch.delete(id.to_s)
+        rescue Batch::Error => e
+          raise JobAdapterError, e.message
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -0,0 +1,350 @@
+require "ood_core/refinements/hash_extensions"
+require "json"
+
+class OodCore::Job::Adapters::Kubernetes::Batch
+
+  require "ood_core/job/adapters/kubernetes/helper"
+
+  Helper = OodCore::Job::Adapters::Kubernetes::Helper
+  Resources = OodCore::Job::Adapters::Kubernetes::Resources
+
+  using OodCore::Refinements::HashExtensions
+
+  class Error < StandardError; end
+
+  attr_reader :config_file, :bin, :cluster_name, :mounts
+  attr_reader :all_namespaces, :using_context, :helper
+  attr_reader :username_prefix
+
+  def initialize(options = {}, helper = Helper.new)
+    options = options.to_h.symbolize_keys
+
+    @config_file = options.fetch(:config_file, default_config_file)
+    @bin = options.fetch(:bin, '/usr/bin/kubectl')
+    @cluster_name = options.fetch(:cluster_name, 'open-ondemand')
+    @mounts = options.fetch(:mounts, []).map { |m| m.to_h.symbolize_keys }
+    @all_namespaces = options.fetch(:all_namespaces, false)
+    @username_prefix = options.fetch(:username_prefix, nil)
+
+    @using_context = false
+    @helper = helper
+
+    begin
+      make_kubectl_config(options)
+    rescue
+      # FIXME could use a log here
+      # means you couldn't 'kubectl set config'
+    end
+  end
+
+  def resource_file(resource_type = 'pod')
+    File.dirname(__FILE__) + "/templates/#{resource_type}.yml.erb"
+  end
+
+  def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+    raise ArgumentError, 'Must specify the script' if script.nil?
+
+    resource_yml, id = generate_id_yml(script.native)
+    call("#{formatted_ns_cmd} create -f -", stdin: resource_yml)
+
+    id
+  end
+
+  def generate_id(name)
+    # 2_821_109_907_456 = 36**8
+    name.downcase.tr(' ', '-') + '-' + rand(2_821_109_907_456).to_s(36)
+  end
+
+  def info_all(attrs: nil)
+    cmd = if all_namespaces
+            "#{base_cmd} get pods -o json --all-namespaces"
+          else
+            "#{namespaced_cmd} get pods -o json"
+          end
+
+    output = call(cmd)
+    all_pods_to_info(output)
+  end
+
+  def info_where_owner(owner, attrs: nil)
+    owner = Array.wrap(owner).map(&:to_s)
+
+    # must at least have job_owner to filter by job_owner
+    attrs = Array.wrap(attrs) | [:job_owner] unless attrs.nil?
+
+    info_all(attrs: attrs).select { |info| owner.include? info.job_owner }
+  end
+
+  def info_all_each(attrs: nil)
+    return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+    info_all(attrs: attrs).each do |job|
+      yield job
+    end
+  end
+
+  def info_where_owner_each(owner, attrs: nil)
+    return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+    info_where_owner(owner, attrs: attrs).each do |job|
+      yield job
+    end
+  end
+
+  def info(id)
+    pod_json = call_json_output('get', 'pod', id)
+
+    begin
+      service_json = call_json_output('get', 'service', service_name(id))
+      secret_json = call_json_output('get', 'secret', secret_name(id))
+    rescue
+      # it's ok if these don't exist
+      service_json ||= nil
+      secret_json ||= nil
+    end
+
+    helper.info_from_json(pod_json: pod_json, service_json: service_json, secret_json: secret_json)
+  end
+
+  def status(id)
+    info(id).status
+  end
+
+  def delete(id)
+    call("#{namespaced_cmd} delete pod #{id}")
+
+    begin
+      call("#{namespaced_cmd} delete service #{service_name(id)}")
+      call("#{namespaced_cmd} delete secret #{secret_name(id)}")
+      call("#{namespaced_cmd} delete configmap #{configmap_name(id)}")
+    rescue
+      # FIXME: retries? delete if exists?
+      # just eat the results of deleting services and secrets
+    end
+  end
+
+  def configmap_mount_path
+    '/ood'
+  end
+
+  private
+
+  # helper to help format multi-line yaml data from the submit.yml into 
+  # mutli-line yaml in the pod.yml.erb
+  def config_data_lines(data)
+    output = []
+    first = true
+
+    data.to_s.each_line do |line|
+      output.append(first ? line : line.prepend("    "))
+      first = false
+    end
+
+    output
+  end
+
+  def username
+    @username ||= Etc.getlogin
+  end
+
+  def k8s_username
+    username_prefix.nil? ? username : "#{username_prefix}-#{username}"
+  end
+
+  def run_as_user
+    Etc.getpwnam(username).uid
+  end
+
+  def run_as_group
+    Etc.getpwnam(username).gid
+  end
+
+  def fs_group
+    run_as_group
+  end
+
+  # helper to template resource yml you're going to submit and
+  # create an id.
+  def generate_id_yml(native_data)
+    container = helper.container_from_native(native_data[:container])
+    id = generate_id(container.name)
+    configmap = helper.configmap_from_native(native_data, id)
+    init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
+    spec = Resources::PodSpec.new(container, init_containers: init_containers)
+    all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
+
+    template = ERB.new(File.read(resource_file))
+
+    [template.result(binding), id]
+  end
+
+  # helper to call kubectl and get json data back.
+  # verb, resrouce and id are the kubernetes parlance terms.
+  # example: 'kubectl get pod my-pod-id' is verb=get, resource=pod
+  # and  id=my-pod-id
+  def call_json_output(verb, resource, id, stdin: nil)
+    cmd = "#{formatted_ns_cmd} #{verb} #{resource} #{id}"
+    data = call(cmd, stdin: stdin)
+    data = data.empty? ? '{}' : data
+    json_data = JSON.parse(data, symbolize_names: true)
+
+    json_data
+  end
+
+  def service_name(id)
+    helper.service_name(id)
+  end
+
+  def secret_name(id)
+    helper.secret_name(id)
+  end
+
+  def configmap_name(id)
+    helper.configmap_name(id)
+  end
+
+  def namespace
+    default_namespace
+  end
+
+  def default_namespace
+    username
+  end
+
+  def context
+    cluster_name
+  end
+
+  def default_config_file
+    (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
+  end
+
+  def default_auth
+    {
+      type: 'managaged'
+    }.symbolize_keys
+  end
+
+  def default_server
+    {
+      endpoint: 'https://localhost:8080',
+      cert_authority_file: nil
+    }.symbolize_keys
+  end
+
+  def formatted_ns_cmd
+    "#{namespaced_cmd} -o json"
+  end
+
+  def namespaced_cmd
+    "#{base_cmd} --namespace=#{namespace}"
+  end
+
+  def base_cmd
+    base = "#{bin} --kubeconfig=#{config_file}"
+    base << " --context=#{context}" if using_context
+    base
+  end
+
+  def all_pods_to_info(data)
+    json_data = JSON.parse(data, symbolize_names: true)
+    pods = json_data.dig(:items)
+
+    info_array = []
+    pods.each do |pod|
+      info = pod_info_from_json(pod)
+      info_array.push(info) unless info.nil?
+    end
+
+    info_array
+  rescue JSON::ParserError
+    # 'no resources in <namespace>' throws parse error
+    []
+  end
+
+  def pod_info_from_json(pod)
+    hash = helper.pod_info_from_json(pod)
+    OodCore::Job::Info.new(hash)
+  rescue Helper::K8sDataError
+    # FIXME: silently eating error, could probably use a logger
+    nil
+  end
+
+  def make_kubectl_config(config)
+    set_cluster(config.fetch(:server, default_server).to_h.symbolize_keys)
+    configure_auth(config.fetch(:auth, default_auth).to_h.symbolize_keys)
+  end
+
+  def configure_auth(auth)
+    type = auth.fetch(:type)
+    return if managed?(type)
+
+    case type
+    when 'gke'
+      set_gke_config(auth)
+    when 'oidc'
+      set_context
+    end
+  end
+
+  def use_context
+    @using_context = true
+  end
+
+  def managed?(type)
+    if type.nil?
+      true # maybe should be false?
+    else
+      type.to_s == 'managed'
+    end
+  end
+
+  def set_gke_config(auth)
+    cred_file = auth.fetch(:svc_acct_file)
+
+    cmd = "gcloud auth activate-service-account --key-file=#{cred_file}"
+    call(cmd)
+
+    set_gke_credentials(auth)
+  end
+
+  def set_gke_credentials(auth)
+
+    zone = auth.fetch(:zone, nil)
+    region = auth.fetch(:region, nil)
+
+    locale = ''
+    locale = "--zone=#{zone}" unless zone.nil?
+    locale = "--region=#{region}" unless region.nil?
+
+    # gke cluster name can probably can differ from what ood calls the cluster
+    cmd = "gcloud container clusters get-credentials #{locale} #{cluster_name}"
+    env = { 'KUBECONFIG' => config_file }
+    call(cmd, env)
+  end
+
+  def set_context
+    cmd = "#{base_cmd} config set-context #{cluster_name}"
+    cmd << " --cluster=#{cluster_name} --namespace=#{namespace}"
+    cmd << " --user=#{k8s_username}"
+
+    call(cmd)
+    use_context
+  end
+
+  def set_cluster(config)
+    server = config.fetch(:endpoint)
+    cert = config.fetch(:cert_authority_file, nil)
+
+    cmd = "#{base_cmd} config set-cluster #{cluster_name}"
+    cmd << " --server=#{server}"
+    cmd << " --certificate-authority=#{cert}" unless cert.nil?
+
+    call(cmd)
+  end
+
+  def call(cmd = '', env: {}, stdin: nil)
+    o, error, s = Open3.capture3(env, cmd, stdin_data: stdin.to_s)
+    s.success? ? o : raise(Error, error)
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -0,0 +1,298 @@
+class OodCore::Job::Adapters::Kubernetes::Helper
+
+  require 'ood_core/job/adapters/kubernetes/resources'
+  require 'resolv'
+  require 'base64'
+
+  class K8sDataError < StandardError; end
+
+  Resources = OodCore::Job::Adapters::Kubernetes::Resources
+
+  # Extract info from json data. The data is expected to be from the kubectl
+  # command and conform to kubernetes' datatype structures.
+  #
+  # Returns { native: {host: localhost, port:80, password: sshhh }} in the info
+  # object field in lieu of writing a connection.yml
+  #
+  # @param pod_json [#to_h]
+  #   the pod data returned from 'kubectl get pod abc-123'
+  # @param service_json [#to_h]
+  #   the service data returned from 'kubectl get service abc-123-service'
+  # @param secret_json [#to_h]
+  #   the secret data returned from 'kubectl get secret abc-123-secret'
+  # @return [OodCore::Job::Info]
+  def info_from_json(pod_json: nil, service_json: nil, secret_json: nil)
+    pod_hash = pod_info_from_json(pod_json)
+    service_hash = service_info_from_json(service_json)
+    secret_hash = secret_info_from_json(secret_json)
+
+    # can't just use deep_merge bc we don't depend *directly* on rails
+    pod_hash[:native] = pod_hash[:native].merge(service_hash[:native])
+    pod_hash[:native] = pod_hash[:native].merge(secret_hash[:native])
+    OodCore::Job::Info.new(pod_hash)
+  rescue NoMethodError
+    raise K8sDataError, "unable to read data correctly from json"
+  end
+
+  # Turn a container hash into a Kubernetes::Resources::Container
+  #
+  # @param container [#to_h]
+  #   the input container hash
+  # @return  [OodCore::Job::Adapters::Kubernetes::Resources::Container]
+  def container_from_native(container)
+    Resources::Container.new(
+      container[:name],
+      container[:image],
+      command: parse_command(container[:command]),
+      port: container[:port],
+      env: container.fetch(:env, []),
+      memory: container[:memory],
+      cpu: container[:cpu],
+      working_dir: container[:working_dir],
+      restart_policy: container[:restart_policy]
+    )
+  end
+
+  # Parse a command string given from a user and return an array.
+  # If given an array, the input is simply returned back.
+  #
+  # @param cmd [#to_s]
+  #   the command to parse
+  # @return [Array<#to_s>]
+  #   the command parsed into an array of arguements
+  def parse_command(cmd)
+    if cmd&.is_a?(Array)
+      cmd
+    else
+      Shellwords.split(cmd.to_s)
+    end
+  end
+
+  # Turn a configmap hash into a Kubernetes::Resources::ConfigMap
+  # that can be used in templates. Needs an id so that the resulting
+  # configmap has a known name.
+  #
+  # @param native [#to_h]
+  #   the input configmap hash
+  # @param id [#to_s]
+  #   the id to use for giving the configmap a name
+  # @return  [OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap]
+  def configmap_from_native(native, id)
+    configmap = native.fetch(:configmap, nil)
+    return nil if configmap.nil?
+
+    Resources::ConfigMap.new(
+      configmap_name(id),
+      configmap[:filename],
+      configmap[:data]
+    )
+  end
+
+  # parse initialization containers from native data
+  #
+  # @param native_data [#to_h]
+  #   the native data to parse. Expected key init_ctrs and for that
+  #   key to be an array of hashes.
+  # @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
+  #   the array of init containers
+  def init_ctrs_from_native(ctrs)
+    init_ctrs = []
+
+    ctrs&.each do |ctr_raw|
+      ctr = container_from_native(ctr_raw)
+      init_ctrs.push(ctr)
+    end
+
+    init_ctrs
+  end
+
+  def service_name(id)
+    id + '-service'
+  end
+
+  def secret_name(id)
+    id + '-secret'
+  end
+
+  def configmap_name(id)
+    id + '-configmap'
+  end
+
+  # Extract pod info from json data. The data is expected to be from the kubectl
+  # command and conform to kubernetes' datatype structures.
+  #
+  # @param json_data [#to_h]
+  #   the pod data returned from 'kubectl get pod abc-123'
+  # @return [#to_h]
+  #   the hash of info expected from adapters
+  def pod_info_from_json(json_data)
+    {
+      id: json_data.dig(:metadata, :name).to_s,
+      job_name: name_from_metadata(json_data.dig(:metadata)),
+      status: pod_status_from_json(json_data),
+      job_owner: json_data.dig(:metadata, :namespace).to_s,
+      submission_time: submission_time(json_data),
+      dispatch_time: dispatch_time(json_data),
+      wallclock_time: wallclock_time(json_data),
+      native: {
+        host: get_host(json_data.dig(:status, :hostIP))
+      },
+      procs: procs_from_json(json_data)
+    }
+  rescue NoMethodError
+    # gotta raise an error because Info.new will throw an error if id is undefined
+    raise K8sDataError, "unable to read data correctly from json"
+  end
+
+  private
+
+  def get_host(ip)
+    Resolv.getname(ip)
+  rescue Resolv::ResolvError
+    ip
+  end
+
+  def name_from_metadata(metadata)
+    name = metadata.dig(:labels, :'app.kubernetes.io/name')
+    name = metadata.dig(:labels, :'k8s-app') if name.nil?
+    name = metadata.dig(:name) if name.nil? # pod-id but better than nil?
+    name
+  end
+
+  def service_info_from_json(json_data)
+    # all we need is the port - .spec.ports[0].nodePort
+    ports = json_data.dig(:spec, :ports)
+    {
+      native:
+        {
+          port: ports[0].dig(:nodePort)
+        }
+    }
+  rescue
+    empty_native
+  end
+
+  def secret_info_from_json(json_data)
+    raw = json_data.dig(:data, :password)
+    {
+      native:
+        {
+          password: Base64.decode64(raw)
+        }
+    }
+  rescue
+    empty_native
+  end
+
+  def empty_native
+    {
+      native: {}
+    }
+  end
+
+  def dispatch_time(json_data)
+    status = pod_status_from_json(json_data)
+    return nil if status == 'undetermined'
+
+    state_data = json_data.dig(:status, :containerStatuses)[0].dig(:state)
+    date_string = nil
+
+    if status == 'completed'
+      date_string = state_data.dig(:terminated, :startedAt)
+    elsif status == 'running'
+      date_string = state_data.dig(:running, :startedAt)
+    end
+
+    date_string.nil? ? nil : DateTime.parse(date_string).to_time.to_i
+  end
+
+  def wallclock_time(json_data)
+    status = pod_status_from_json(json_data)
+    return nil if status == 'undetermined'
+
+    state_data = json_data.dig(:status, :containerStatuses)[0].dig(:state)
+    start_time = dispatch_time(json_data)
+    return nil if start_time.nil?
+
+    et = end_time(status, state_data)
+
+    et.nil? ? nil : et - start_time
+  end
+
+  def end_time(status, state_data)
+    if status == 'completed'
+      end_time_string = state_data.dig(:terminated, :finishedAt)
+      et = DateTime.parse(end_time_string).to_time.to_i
+    elsif status == 'running'
+      et = DateTime.now.to_time.to_i
+    else
+      et = nil
+    end
+
+    et
+  end
+
+  def submission_time(json_data)
+    status = json_data.dig(:status)
+    start = status.dig(:startTime)
+
+    if start.nil?
+      # the pod is in some pending state limbo
+      conditions = status.dig(:conditions)
+      # best guess to start time is just the first condition's
+      # transition time
+      str = conditions[0].dig(:lastTransitionTime)
+    else
+      str = start
+    end
+
+    DateTime.parse(str).to_time.to_i
+  end
+
+  def pod_status_from_json(json_data)
+    state = 'undetermined'
+    status = json_data.dig(:status)
+    container_statuses = status.dig(:containerStatuses)
+
+    if container_statuses.nil?
+      # if you're here, it means you're pending, probably unschedulable
+      return OodCore::Job::Status.new(state: state)
+    end
+
+    # only support 1 container/pod
+    json_state = container_statuses[0].dig(:state)
+    state = 'running' unless json_state.dig(:running).nil?
+    state = terminated_state(json_state) unless json_state.dig(:terminated).nil?
+    state = 'queued' unless json_state.dig(:waiting).nil?
+
+    OodCore::Job::Status.new(state: state)
+  end
+
+  def terminated_state(status)
+    reason = status.dig(:terminated, :reason)
+    if reason == 'Error'
+      'suspended'
+    else
+      'completed'
+    end
+  end
+
+  def procs_from_json(json_data)
+    containers = json_data.dig(:spec, :containers)
+    resources = containers[0].dig(:resources)
+
+    cpu = resources.dig(:limits, :cpu)
+    millicores_rex = /(\d+)m/
+
+    # ok to return string bc nil.to_i == 0 and we'd rather return
+    # nil (undefined) than 0 which is confusing.
+    if millicores_rex.match?(cpu)
+      millicores =  millicores_rex.match(cpu)[1].to_i
+
+      # have to return at least 1 bc 200m could be 0
+      ((millicores + 1000) / 1000).to_s
+    else
+      cpu
+    end
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/resources.rb

@@ -0,0 +1,56 @@
+module OodCore::Job::Adapters::Kubernetes::Resources
+
+  class ConfigMap
+    attr_accessor :name, :filename, :data
+
+    def initialize(name, filename, data)
+      @name = name
+      @filename = filename
+      @data = data
+    end
+  end
+
+  class Container
+    attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
+                  :restart_policy
+
+    def initialize(
+        name, image, command: [], port: nil, env: [], memory: "4Gi", cpu: "1",
+        working_dir: "", restart_policy: "Never"
+      )
+      raise ArgumentError, "containers need valid names and images" unless name && image
+
+      @name = name
+      @image = image
+      @command = command.nil? ? [] : command
+      @port = port&.to_i
+      @env = env.nil? ? [] : env
+      @memory = memory.nil? ? "4Gi" : memory
+      @cpu = cpu.nil? ? "1" : cpu
+      @working_dir = working_dir.nil? ? "" : working_dir
+      @restart_policy = restart_policy.nil? ? "Never" : restart_policy
+    end
+
+    def ==(other)
+      name == other.name &&
+        image == other.image &&
+        command == other.command &&
+        port == other.port &&
+        env == other.env &&
+        memory == other.memory &&
+        cpu == other.cpu &&
+        working_dir == other.working_dir &&
+        restart_policy == other.restart_policy
+    end
+
+  end
+
+  class PodSpec
+    attr_accessor :container, :init_containers
+    def initialize(container, init_containers: nil)
+      @container = container
+      @init_containers = init_containers
+    end
+  end
+  
+end

data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb

@@ -0,0 +1,123 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: <%= namespace %>
+  name: <%= id %>
+  labels:
+    job: <%= id %>
+    app.kubernetes.io/name: <%= container.name %>
+    app.kubernetes.io/managed-by: open-ondemand
+spec:
+  restartPolicy: <%= spec.container.restart_policy %>
+  securityContext:
+    runAsUser: <%= run_as_user %>
+    runAsGroup: <%= run_as_group %>
+    fsGroup: <%= fs_group %>
+  containers:
+  - name: "<%= spec.container.name %>"
+    image: <%= spec.container.image %>
+    imagePullPolicy: IfNotPresent
+    <% unless spec.container.working_dir.empty? %>
+    workingDir: "<%= spec.container.working_dir %>"
+    <% end %>
+    <% unless spec.container.env.empty? %>
+    env:
+    <% spec.container.env.each do |env| %>
+    - name: <%= env[:name] %>
+      value: "<%= env[:value] %>"
+    <% end %> <%# for each env %>
+    <% end %> <%# unless env is nil %>
+    <% unless spec.container.command.empty? %>
+    command:
+    <% spec.container.command.each do |cmd| %>
+    - "<%= cmd %>"
+    <% end %> <%# for each command %>
+    <% end %> <%# unless command is nil %>
+    <% unless spec.container.port.nil? %>
+    ports:
+    - containerPort: <%= spec.container.port %>
+    <% end %> 
+    volumeMounts:
+    <% unless configmap.nil? %>
+    - name: configmap-volume
+      mountPath:  <%= configmap_mount_path %>
+    <% end %>
+    <% all_mounts.each do |mount| %>
+    - name: <%= mount[:name] %>
+      mountPath: <%= mount[:destination_path] %>
+    <% end %> <%# for each mount %>
+    resources:
+      limits:
+        memory: "<%= spec.container.memory %>"
+        cpu: "<%= spec.container.cpu %>"
+      requests:
+        memory: "<%= spec.container.memory %>"
+        cpu: "<%= spec.container.cpu %>"
+  <% unless spec.init_containers.nil? %>
+  initContainers:
+  <% spec.init_containers.each do |ctr| %>
+  - name: "<%= ctr.name %>"
+    image: "<%= ctr.image %>"
+    command:
+    <% ctr.command.each do |cmd| %>
+    - "<%= cmd %>"
+    <% end %> <%# command loop  %>
+    volumeMounts:
+    <% unless configmap.nil? %>
+    - name: configmap-volume
+      mountPath:  <%= configmap_mount_path %>
+    <% end %>
+    <% all_mounts.each do |mount| %>
+    - name: <%= mount[:name] %>
+      mountPath: <%= mount[:destination_path] %>
+    <% end %> <%# for each mount %>
+  <% end %> <%# init container loop  %>
+  <% end %> <%# if init containers  %>
+  <% unless configmap.nil? || all_mounts.empty? %>
+  volumes:
+  <% end %> <%# configmap.nil? || all_mounts.empty? %>
+  <% unless configmap.nil? %>
+  - name: configmap-volume
+    configMap:
+      name: <%= configmap_name(id) %>
+  <% end %>
+  <% all_mounts.each do |mount| %>
+  <% if mount[:type] == 'nfs' %>
+  - name: <%= mount[:name] %>
+    nfs:
+      server: <%= mount[:host] %>
+      path: <%= mount[:path] %>
+  <% elsif mount[:type] == 'host' %>
+  - name: <%= mount[:name] %>
+    hostPath:
+      path: <%= mount[:path] %>
+      type: <%= mount[:host_type] %>
+  <% end %> <%# if mount is [host,nfs] %>
+  <% end %> <%# for each mount %>
+---
+<% unless spec.container.port.nil? %>
+apiVersion: v1
+kind: Service
+metadata:
+  name: <%= service_name(id) %>
+  namespace: <%= namespace %>
+spec:
+  selector:
+    job: <%= id %>
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: <%= spec.container.port %>
+  type: NodePort
+<% end %> <%# end for service %>
+---
+<% unless configmap.nil? %>
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: <%= configmap_name(id) %>
+  namespace: <%= namespace %>
+data:
+  <%= configmap.filename %>: |
+    <% config_data_lines(configmap.data).each do |line| %><%= line %><% end %>
+<% end %> <%# end for configmap %>

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -166,7 +166,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'email_on_terminated' => script_email_on_event(script, 'terminated'),
         'email_on_start' => script_email_on_event(script, 'started'),
         'environment' => export_env(script),
-        'error_path' => (script.error_path) ? script.error_path.to_s : '/dev/null',
+        'error_path' => error_path(script),
         'job_name' => script.job_name.to_s,
         'output_path' => (script.output_path) ? script.output_path.to_s : '/dev/null',
         'script_content' => content,
@@ -176,6 +176,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'singularity_image' => singularity_image(script.native),
         'ssh_hosts' => ssh_hosts,
         'tmux_bin' => tmux_bin,
+        'workdir' => (script.workdir) ? script.workdir.to_s : '/tmp',
       }.each{
         |key, value| bnd.local_variable_set(key, value)
       }
@@ -272,4 +273,11 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     return false if script.content.empty?
     script.content.split("\n").first.start_with?('#!/')
   end
+
+  def error_path(script)
+    return script.error_path.to_s if script.error_path
+    return script.output_path.to_s if script.output_path
+
+    '/dev/null'
+  end
 end

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -16,13 +16,9 @@ fi
 echo $hostname
 
 # Put the script into a temp file on localhost
-<% if debug %>
-singularity_tmp_file=$(mktemp -p "$HOME" --suffix '_sing')
-tmux_tmp_file=$(mktemp -p "$HOME" --suffix "_tmux")
-<% else %>
-singularity_tmp_file=$(mktemp)
-tmux_tmp_file=$(mktemp)
-<% end %>
+singularity_tmp_file=$(mktemp -p "<%= workdir %>" --suffix '_sing')
+tmux_tmp_file=$(mktemp -p "<%= workdir %>" --suffix "_tmux")
+
 
 # Create an executable to run in a tmux session
 # The escaped HEREDOC means that we need to substitute in $singularity_tmp_file ourselves
@@ -69,10 +65,3 @@ SINGULARITY_LAUNCHER
 chmod +x "$singularity_tmp_file"
 chmod +x "$tmux_tmp_file"
 <%= tmux_bin %> new-session -d -s "<%= session_name %>" "$tmux_tmp_file"
-
-# Remove the file
-<% if ! debug %>
-# Wait 1 second to ensure that tmux session has started before the file is removed
-sleep 1
-rm -f "$tmux_tmp_file"; rm -f "$singularity_tmp_file"
-<% end %>

data/lib/ood_core/job/adapters/slurm.rb

@@ -80,6 +80,9 @@ module OodCore
           # from
           class Error < StandardError; end
 
+          # An error indicating the slurm command timed out
+          class SlurmTimeoutError < Error; end
+
           # @param cluster [#to_s, nil] the cluster name
           # @param conf [#to_s, nil] path to the slurm conf
           # @param bin [#to_s] path to slurm installation binaries
@@ -147,6 +150,9 @@ module OodCore
               end
               jobs
             end
+          rescue SlurmTimeoutError
+            # TODO: could use a log entry here
+            return [{ id: id, state: 'undetermined' }]
           end
 
           def squeue_fields(attrs)
@@ -303,7 +309,18 @@ module OodCore
 
               cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking)
               o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin.to_s)
-              s.success? ? o : raise(Error, e)
+              s.success? ? interpret_and_raise(o, e) : raise(Error, e)
+            end
+
+            # Helper function to raise an error based on the contents of stderr.
+            # Slurm exits 0 even when the command fails, so we need to interpret stderr
+            # to see if the command was actually successful.
+            def interpret_and_raise(stdout, stderr)
+              return stdout if stderr.empty?
+
+              raise SlurmTimeoutError, stderr if /^slurm_load_jobs error: Socket timed out/.match(stderr)
+
+              stdout
             end
 
             def squeue_attrs_for_info_attrs(attrs)

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.13.0"
+  VERSION = "0.14.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -166,6 +166,11 @@ files:
 - lib/ood_core/job/adapters/ccq.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
+- lib/ood_core/job/adapters/kubernetes.rb
+- lib/ood_core/job/adapters/kubernetes/batch.rb
+- lib/ood_core/job/adapters/kubernetes/helper.rb
+- lib/ood_core/job/adapters/kubernetes/resources.rb
+- lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb
 - lib/ood_core/job/adapters/linux_host.rb
 - lib/ood_core/job/adapters/linux_host/launcher.rb
 - lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
@@ -216,7 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Open OnDemand core library