ood_core 0.14.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52ba764b085dedb7eaeb06d95751f1804a50488e1859f980a7836d2d9032b95d
-  data.tar.gz: c2dc5edf395fe158960f33b80c554f3dc745f15e7ec1337b738683a0e1bbdc7f
+  metadata.gz: 266e24cd2e11c2c712b6128bdc3f82bf9781ae9885f8f0eb21439cb80c889c90
+  data.tar.gz: 01e682b6313468371076cdfd4ff2df2f3c06c661af9b4d7d7a65b7dcf3e2d836
 SHA512:
-  metadata.gz: 59915bae23a008a923c249d222e50548a7bee3438144068a29ae1cafdd489ca1229ee1a14f4f81e3fd065381f46f920bef24344fe633c7c578cb1f6a4f9a2a77
-  data.tar.gz: 8d2ca42c7f49158c8d321c21b79aff1c636df3c77bb7e71107db70371a34058d79c8a5ec32ca93883e7d3bcc7dc2202375144d23613f167ab089318d6270248c
+  metadata.gz: 22721c9d368ec44533d93914f977576ee77786ad0926976fa24067c7f353104edd32baffac34723b730e7f711c5b7581cf4f72f6232d050c719edcfc1b3cb14f
+  data.tar.gz: 5d99a3c782aad5e420333653e51073cdb6535b7689053452da7ee7ef234292769e70d4010d751aacd4576daf531700136c2aa0ef7a2efc31edcb29e4c01c8be4

data/.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Unit Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Ruby using Bundler
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "2.7.1"
+          bundler-cache: true
+          bundler: "2.1.4"
+
+      - name: install gems
+        run: bundle install
+
+      - name: test
+        run: bundle exec rake spec

data/CHANGELOG.md

@@ -6,6 +6,87 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## [0.17.0] - 5-26-2021
+
+### Fixed
+
+- All Kubernetes resources now have the same labels in [280](https://github.com/OSC/ood_core/pull/280).
+- Kubernetes does not crash when no configmap is defined in [282](https://github.com/OSC/ood_core/pull/282).
+- Kubernetes will not specify init containers if there are none in
+  [284](https://github.com/OSC/ood_core/pull/284).
+
+### Added
+
+- Kubernetes, Slurm and Torque now support the script option `gpus_per_node` in
+  [266](https://github.com/OSC/ood_core/pull/266).
+- Kubernetes will now save the pod.yml into the staged root in
+  [277](https://github.com/OSC/ood_core/pull/277).
+- Kubernetes now allows for node selector in [264](https://github.com/OSC/ood_core/pull/264).
+- Kubernetes pods now have access the environment variable POD_NAMESPACE in
+  [275](https://github.com/OSC/ood_core/pull/275).
+- Kubernetes pods can now specify the image pull policy in [272](https://github.com/OSC/ood_core/pull/272).
+- Cluster config's batch_connect now support `ssh_allow` to disable sshing to compute
+  nodes per cluster in [286](https://github.com/OSC/ood_core/pull/286).
+- Kubernetes will now add the templated script content to a configmap in
+  [273](https://github.com/OSC/ood_core/pull/273).
+
+### Changed
+
+- Kubernetes username prefix no longer appends a - in [271](https://github.com/OSC/ood_core/pull/271).
+
+
+
+## [0.16.1] - 2021-04-23
+### Fixed
+- memorized some allow? variables to have better support around ACLS in
+  [267](https://github.com/OSC/ood_core/pull/267)
+
+## [0.16.0] - 2021-04-20
+### Fixed
+- tmux 2.7+ bug in the linux host adapter in [2.5.8](https://github.com/OSC/ood_core/pull/258)
+  and [259](https://github.com/OSC/ood_core/pull/259).
+
+### Changed
+
+- Changed how k8s configmaps in are defined in [251](https://github.com/OSC/ood_core/pull/251).
+  The data structure now expects a key called files which is an array of objects that hold
+  filename, data, mount_path, sub_path and init_mount_path.
+  [255](https://github.com/OSC/ood_core/pull/255) also relates to this interface change.
+
+### Added
+
+- The k8s adapter can now specify environment variables and creates defaults 
+  in [252](https://github.com/OSC/ood_core/pull/252).
+- The k8s adapter can now specify image pull secrets in [253](https://github.com/OSC/ood_core/pull/253).
+
+## [0.15.1] - 2021-02-25
+### Fixed
+- kubernetes adapter uses the full module for helpers in [245](https://github.com/OSC/ood_core/pull/245).
+
+### Changed
+- kubernetes pods spawn with runAsNonRoot set to true in [247](https://github.com/OSC/ood_core/pull/247).
+- kubernetes pods can spawn with supplemental groups along with some other in security defaults in
+  [246](https://github.com/OSC/ood_core/pull/246).
+
+## [0.15.0] - 2021-01-26
+### Fixed
+- ccq adapter now accepts job names with spaces in [210](https://github.com/OSC/ood_core/pull/209)
+- k8s correctly handles having no mount volumes in [239](https://github.com/OSC/ood_core/pull/239)
+
+### Added
+- k8s adapter now applies account metadata to resources in [216](https://github.com/OSC/ood_core/pull/216) and
+  [231](https://github.com/OSC/ood_core/pull/231)
+- k8s adapter can now prefix namespaces in [218](https://github.com/OSC/ood_core/pull/218)
+- k8s adapter now applies time limits to pods in [224](https://github.com/OSC/ood_core/pull/224)
+
+### Changed
+- testing automation is now done in github actions in [221](https://github.com/OSC/ood_core/pull/218)
+- update bunlder to 2.1.4 and ruby to 2.7 in [235](https://github.com/OSC/ood_core/pull/218) updated bundler and ruby
+- k8s adapter more appropriately labels unschedulable pods as queued in [230](https://github.com/OSC/ood_core/pull/230)
+- k8s adapter now uses the script#ood_connection_info API instead of script#native in
+  [222](https://github.com/OSC/ood_core/pull/222)
+
 ## [0.14.0] - 2020-10-01
 ### Added 
 - Kubernetes adapter in PR [156](https://github.com/OSC/ood_core/pull/156)
@@ -255,7 +336,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.14.0...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.16.0...HEAD
+[0.17.0]: https://github.com/OSC/ood_core/compare/v0.16.1...v0.17.0
+[0.16.1]: https://github.com/OSC/ood_core/compare/v0.16.0...v0.16.1
+[0.16.0]: https://github.com/OSC/ood_core/compare/v0.15.1...v0.16.0
+[0.15.1]: https://github.com/OSC/ood_core/compare/v0.15.0...v0.15.1
+[0.15.0]: https://github.com/OSC/ood_core/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0
 [0.13.0]: https://github.com/OSC/ood_core/compare/v0.12.0...v0.13.0
 [0.12.0]: https://github.com/OSC/ood_core/compare/v0.11.4...v0.12.0

data/README.md

@@ -1,6 +1,6 @@
 # OodCore
 
-[![Build Status](https://travis-ci.org/OSC/ood_core.svg?branch=master)](https://travis-ci.org/OSC/ood_core)
+[![Build Status](https://github.com/osc/ood_core/workflows/Unit%20Tests/badge.svg)](https://github.com/OSC/ood_core/actions?query=workflow%3A%22Unit+Tests%22)
 ![GitHub Release](https://img.shields.io/github/release/osc/ood_core.svg)
 ![GitHub License](https://img.shields.io/github/license/osc/ood_core.svg)
 

data/lib/ood_core/cluster.rb

@@ -78,7 +78,9 @@ module OodCore
     # Whether the login feature is allowed
     # @return [Boolean] is login allowed
     def login_allow?
-      allow? && !login_config.empty?
+      return @login_allow if defined?(@login_allow)
+
+      @login_allow = (allow? && !login_config.empty?)
     end
 
     # Build a job adapter from the job configuration
@@ -90,9 +92,11 @@ module OodCore
     # Whether the job feature is allowed based on the ACLs
     # @return [Boolean] is the job feature allowed
     def job_allow?
-      allow? &&
-        !job_config.empty? &&
-        build_acls(job_config.fetch(:acls, []).map(&:to_h)).all?(&:allow?)
+      return @job_allow if defined?(@job_allow)
+
+      @job_allow = (allow? && ! job_config.empty? && build_acls(
+        job_config.fetch(:acls, []).map(&:to_h)
+      ).all?(&:allow?))
     end
 
     # The batch connect template configuration used for this cluster
@@ -138,7 +142,18 @@ module OodCore
     # Whether this cluster is allowed to be used
     # @return [Boolean] whether cluster is allowed
     def allow?
-      acls.all?(&:allow?)
+      return @allow if defined?(@allow)
+
+      @allow = acls.all?(&:allow?)
+    end
+
+    # Whether this cluster supports SSH to batch connect nodes
+    # @return [Boolean] whether cluster supports SSH to batch connect node
+    def batch_connect_ssh_allow?
+      return @batch_connect_ssh_allow if defined?(@batch_connect_ssh_allow)
+      return @batch_connect_ssh_allow = true if batch_connect_config.nil?
+
+      @batch_connect_ssh_allow = batch_connect_config.fetch(:ssh_allow, true)
     end
 
     # The comparison operator

data/lib/ood_core/job/adapters/ccq.rb

@@ -203,6 +203,10 @@ module OodCore
           'ccq_ood_script_'
         end
 
+        def ccqstat_regex
+          /^(?<id>\S+)\s+(?<name>.+)\s+(?<username>\S+)\s+(?<scheduler>\S+)\s+(?<status>\S+)\s*$/
+        end
+
         def parse_job_id_from_ccqsub(output)
           match_data = /#{jobid_regex}/.match(output)
           # match_data could be nil, OR re-configured jobid_regex could be looking for a different named group
@@ -236,28 +240,31 @@ module OodCore
         def info_from_ccqstat(data)
           infos = []
 
-          data.to_s.each_line do |line|
-            words = line.split(/\s/).reject(&:empty?)
-            next if !words.empty? && words[0] == "Id" # just skip the header
-
-            infos << Info.new(line_to_hash(words)) if words.size == 5
+          data.to_s.lines.drop(1).each do |line|
+            match_data = ccqstat_regex.match(line)
+            infos << Info.new(ccqstat_match_to_hash(match_data)) if valid_ccqstat_match?(match_data)
           end
 
           infos
         end
 
-        def line_to_hash(words)
-          return unless words.size == 5
-
+        def ccqstat_match_to_hash(match)
           data_hash = {}
-          data_hash[:id] = words[0]
-          data_hash[:job_name] = words[1]
-          data_hash[:job_owner] = words[2]
-          data_hash[:status] = get_state(words[4])
+          data_hash[:id] = match.named_captures.fetch('id', nil)
+          data_hash[:job_owner] = match.named_captures.fetch('username', nil)
+          data_hash[:status] = get_state(match.named_captures.fetch('status', nil))
+
+          # The regex leaves trailing empty spaces. There's no way to tell if they're _actually_
+          # a part of the job name or not, so we assume they're not and add the rstrip.
+          data_hash[:job_name] = match.named_captures.fetch('name', nil).to_s.rstrip
 
           data_hash
         end
 
+        def valid_ccqstat_match?(match)
+          !match.nil? && !match.named_captures.fetch('id', nil).nil?
+        end
+
         def get_state(state)
           STATE_MAP.fetch(state, :undetermined)
         end

data/lib/ood_core/job/adapters/kubernetes.rb

@@ -7,7 +7,7 @@ module OodCore
       using Refinements::HashExtensions
 
       def self.build_kubernetes(config)
-        batch = Adapters::Kubernetes::Batch.new(config.to_h.symbolize_keys, Adapters::Kubernetes::Helper.new)
+        batch = Adapters::Kubernetes::Batch.new(config.to_h.symbolize_keys)
         Adapters::Kubernetes.new(batch)
       end
     end

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -3,31 +3,31 @@ require "json"
 
 class OodCore::Job::Adapters::Kubernetes::Batch
 
-  require "ood_core/job/adapters/kubernetes/helper"
-
-  Helper = OodCore::Job::Adapters::Kubernetes::Helper
-  Resources = OodCore::Job::Adapters::Kubernetes::Resources
+  require_relative "helper"
+  require_relative "k8s_job_info"
 
   using OodCore::Refinements::HashExtensions
 
   class Error < StandardError; end
+  class NotFoundError < StandardError; end
 
-  attr_reader :config_file, :bin, :cluster_name, :mounts
+  attr_reader :config_file, :bin, :cluster, :mounts
   attr_reader :all_namespaces, :using_context, :helper
-  attr_reader :username_prefix
+  attr_reader :username_prefix, :namespace_prefix
 
-  def initialize(options = {}, helper = Helper.new)
+  def initialize(options = {})
     options = options.to_h.symbolize_keys
 
     @config_file = options.fetch(:config_file, default_config_file)
     @bin = options.fetch(:bin, '/usr/bin/kubectl')
-    @cluster_name = options.fetch(:cluster_name, 'open-ondemand')
+    @cluster = options.fetch(:cluster, 'open-ondemand')
     @mounts = options.fetch(:mounts, []).map { |m| m.to_h.symbolize_keys }
     @all_namespaces = options.fetch(:all_namespaces, false)
-    @username_prefix = options.fetch(:username_prefix, nil)
+    @username_prefix = options.fetch(:username_prefix, '')
+    @namespace_prefix = options.fetch(:namespace_prefix, '')
 
     @using_context = false
-    @helper = helper
+    @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
 
     begin
       make_kubectl_config(options)
@@ -44,7 +44,10 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
     raise ArgumentError, 'Must specify the script' if script.nil?
 
-    resource_yml, id = generate_id_yml(script.native)
+    resource_yml, id = generate_id_yml(script)
+    if !script.workdir.nil? && Dir.exist?(script.workdir)
+      File.open(File.join(script.workdir, 'pod.yml'), 'w') { |f| f.write resource_yml }
+    end
     call("#{formatted_ns_cmd} create -f -", stdin: resource_yml)
 
     id
@@ -92,16 +95,11 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def info(id)
-    pod_json = call_json_output('get', 'pod', id)
+    pod_json = safe_call('get', 'pod', id)
+    return OodCore::Job::Info.new({ id: id, status: 'completed' }) if pod_json.empty?
 
-    begin
-      service_json = call_json_output('get', 'service', service_name(id))
-      secret_json = call_json_output('get', 'secret', secret_name(id))
-    rescue
-      # it's ok if these don't exist
-      service_json ||= nil
-      secret_json ||= nil
-    end
+    service_json = safe_call('get', 'service', service_name(id))
+    secret_json = safe_call('get', 'secret', secret_name(id))
 
     helper.info_from_json(pod_json: pod_json, service_json: service_json, secret_json: secret_json)
   end
@@ -111,24 +109,27 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def delete(id)
-    call("#{namespaced_cmd} delete pod #{id}")
+    safe_call("delete", "pod", id)
+    safe_call("delete", "service", service_name(id))
+    safe_call("delete", "secret", secret_name(id))
+    safe_call("delete", "configmap", configmap_name(id))
+  end
 
+  private
+
+  def safe_call(verb, resource, id)
     begin
-      call("#{namespaced_cmd} delete service #{service_name(id)}")
-      call("#{namespaced_cmd} delete secret #{secret_name(id)}")
-      call("#{namespaced_cmd} delete configmap #{configmap_name(id)}")
-    rescue
-      # FIXME: retries? delete if exists?
-      # just eat the results of deleting services and secrets
+      case verb.to_s
+      when "get"
+        call_json_output('get', resource, id)
+      when "delete"
+        call("#{namespaced_cmd} delete #{resource} #{id}")
+      end
+    rescue NotFoundError
+      {}
     end
   end
 
-  def configmap_mount_path
-    '/ood'
-  end
-
-  private
-
   # helper to help format multi-line yaml data from the submit.yml into 
   # mutli-line yaml in the pod.yml.erb
   def config_data_lines(data)
@@ -148,32 +149,57 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def k8s_username
-    username_prefix.nil? ? username : "#{username_prefix}-#{username}"
+    "#{username_prefix}#{username}"
+  end
+
+  def user
+    @user ||= Etc.getpwnam(username)
+  end
+
+  def home_dir
+    user.dir
   end
 
   def run_as_user
-    Etc.getpwnam(username).uid
+    user.uid
   end
 
   def run_as_group
-    Etc.getpwnam(username).gid
+    user.gid
   end
 
   def fs_group
     run_as_group
   end
 
+  def group
+    Etc.getgrgid(run_as_group).name
+  end
+
+  def default_env
+    {
+      USER: username,
+      UID: run_as_user,
+      HOME: home_dir,
+      GROUP: group,
+      GID: run_as_group,
+    }
+  end
+
   # helper to template resource yml you're going to submit and
   # create an id.
-  def generate_id_yml(native_data)
-    container = helper.container_from_native(native_data[:container])
+  def generate_id_yml(script)
+    native_data = script.native
+    container = helper.container_from_native(native_data[:container], default_env)
     id = generate_id(container.name)
-    configmap = helper.configmap_from_native(native_data, id)
-    init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
-    spec = Resources::PodSpec.new(container, init_containers: init_containers)
+    configmap = helper.configmap_from_native(native_data, id, script.content)
+    init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
+    spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
     all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
+    node_selector = native_data[:node_selector].nil? ? {} : native_data[:node_selector]
+    gpu_type = native_data[:gpu_type].nil? ? "nvidia.com/gpu" : native_data[:gpu_type]
 
-    template = ERB.new(File.read(resource_file))
+    template = ERB.new(File.read(resource_file), nil, '-')
 
     [template.result(binding), id]
   end
@@ -204,15 +230,11 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def namespace
-    default_namespace
-  end
-
-  def default_namespace
-    username
+    "#{namespace_prefix}#{username}"
   end
 
   def context
-    cluster_name
+    cluster
   end
 
   def default_config_file
@@ -264,7 +286,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
 
   def pod_info_from_json(pod)
     hash = helper.pod_info_from_json(pod)
-    OodCore::Job::Info.new(hash)
+    K8sJobInfo.new(hash)
   rescue Helper::K8sDataError
     # FIXME: silently eating error, could probably use a logger
     nil
@@ -318,14 +340,14 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     locale = "--region=#{region}" unless region.nil?
 
     # gke cluster name can probably can differ from what ood calls the cluster
-    cmd = "gcloud container clusters get-credentials #{locale} #{cluster_name}"
+    cmd = "gcloud container clusters get-credentials #{locale} #{cluster}"
     env = { 'KUBECONFIG' => config_file }
     call(cmd, env)
   end
 
   def set_context
-    cmd = "#{base_cmd} config set-context #{cluster_name}"
-    cmd << " --cluster=#{cluster_name} --namespace=#{namespace}"
+    cmd = "#{base_cmd} config set-context #{cluster}"
+    cmd << " --cluster=#{cluster} --namespace=#{namespace}"
     cmd << " --user=#{k8s_username}"
 
     call(cmd)
@@ -336,7 +358,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     server = config.fetch(:endpoint)
     cert = config.fetch(:cert_authority_file, nil)
 
-    cmd = "#{base_cmd} config set-cluster #{cluster_name}"
+    cmd = "#{base_cmd} config set-cluster #{cluster}"
     cmd << " --server=#{server}"
     cmd << " --certificate-authority=#{cert}" unless cert.nil?
 
@@ -344,7 +366,12 @@ class OodCore::Job::Adapters::Kubernetes::Batch
   end
 
   def call(cmd = '', env: {}, stdin: nil)
-    o, error, s = Open3.capture3(env, cmd, stdin_data: stdin.to_s)
-    s.success? ? o : raise(Error, error)
+    o, e, s = Open3.capture3(env, cmd, stdin_data: stdin.to_s)
+    s.success? ? o : interpret_and_raise(e)
+  end
+
+  def interpret_and_raise(stderr)
+    raise NotFoundError, stderr if /^Error from server \(NotFound\):/.match(stderr)
+    raise(Error, stderr)
   end
 end