ood_core 0.14.0 → 0.17.0

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -1,18 +1,17 @@
 class OodCore::Job::Adapters::Kubernetes::Helper
 
-  require 'ood_core/job/adapters/kubernetes/resources'
+  require_relative 'resources'
+  require_relative 'k8s_job_info'
   require 'resolv'
   require 'base64'
+  require 'active_support/core_ext/hash'
 
   class K8sDataError < StandardError; end
 
-  Resources = OodCore::Job::Adapters::Kubernetes::Resources
-
   # Extract info from json data. The data is expected to be from the kubectl
   # command and conform to kubernetes' datatype structures.
   #
-  # Returns { native: {host: localhost, port:80, password: sshhh }} in the info
-  # object field in lieu of writing a connection.yml
+  # Returns K8sJobInfo in the in lieu of writing a connection.yml
   #
   # @param pod_json [#to_h]
   #   the pod data returned from 'kubectl get pod abc-123'
@@ -20,16 +19,17 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   #   the service data returned from 'kubectl get service abc-123-service'
   # @param secret_json [#to_h]
   #   the secret data returned from 'kubectl get secret abc-123-secret'
-  # @return [OodCore::Job::Info]
-  def info_from_json(pod_json: nil, service_json: nil, secret_json: nil)
-    pod_hash = pod_info_from_json(pod_json)
+  # @param ns_prefix [#to_s]
+  #   the namespace prefix so that namespaces can be converted back to usernames
+  # @return [OodCore::Job::Adapters::Kubernetes::K8sJobInfo]
+  def info_from_json(pod_json: nil, service_json: nil, secret_json: nil, ns_prefix: nil)
+    pod_hash = pod_info_from_json(pod_json, ns_prefix: ns_prefix)
     service_hash = service_info_from_json(service_json)
     secret_hash = secret_info_from_json(secret_json)
 
-    # can't just use deep_merge bc we don't depend *directly* on rails
-    pod_hash[:native] = pod_hash[:native].merge(service_hash[:native])
-    pod_hash[:native] = pod_hash[:native].merge(secret_hash[:native])
-    OodCore::Job::Info.new(pod_hash)
+    pod_hash.deep_merge!(service_hash)
+    pod_hash.deep_merge!(secret_hash)
+    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(pod_hash)
   rescue NoMethodError
     raise K8sDataError, "unable to read data correctly from json"
   end
@@ -38,18 +38,23 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   #
   # @param container [#to_h]
   #   the input container hash
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
   # @return  [OodCore::Job::Adapters::Kubernetes::Resources::Container]
-  def container_from_native(container)
-    Resources::Container.new(
+  def container_from_native(container, default_env)
+    env = container.fetch(:env, {}).to_h.symbolize_keys
+    OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
       container[:name],
       container[:image],
       command: parse_command(container[:command]),
       port: container[:port],
-      env: container.fetch(:env, []),
+      env: default_env.merge(env),
       memory: container[:memory],
       cpu: container[:cpu],
       working_dir: container[:working_dir],
-      restart_policy: container[:restart_policy]
+      restart_policy: container[:restart_policy],
+      image_pull_policy: container[:image_pull_policy],
+      image_pull_secret: container[:image_pull_secret]
     )
   end
 
@@ -76,15 +81,22 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   #   the input configmap hash
   # @param id [#to_s]
   #   the id to use for giving the configmap a name
+  # @param script_content [#to_s]
+  #   the batch script content
   # @return  [OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap]
-  def configmap_from_native(native, id)
-    configmap = native.fetch(:configmap, nil)
-    return nil if configmap.nil?
-
-    Resources::ConfigMap.new(
+  def configmap_from_native(native, id, script_content)
+    configmap = native.fetch(:configmap, {})
+    configmap[:files] ||= []
+    configmap[:files] << {
+      filename: 'script.sh',
+      data: script_content,
+      mount_path: '/ood/script.sh',
+      sub_path: 'script.sh',
+    } unless configmap[:files].any? { |f| f[:filename] == 'script.sh' }
+
+    OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
       configmap_name(id),
-      configmap[:filename],
-      configmap[:data]
+      (configmap[:files] || [])
     )
   end
 
@@ -93,13 +105,15 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   # @param native_data [#to_h]
   #   the native data to parse. Expected key init_ctrs and for that
   #   key to be an array of hashes.
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
   # @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
   #   the array of init containers
-  def init_ctrs_from_native(ctrs)
+  def init_ctrs_from_native(ctrs, default_env)
     init_ctrs = []
 
     ctrs&.each do |ctr_raw|
-      ctr = container_from_native(ctr_raw)
+      ctr = container_from_native(ctr_raw, default_env)
       init_ctrs.push(ctr)
     end
 
@@ -118,25 +132,29 @@ class OodCore::Job::Adapters::Kubernetes::Helper
     id + '-configmap'
   end
 
+  def seconds_to_duration(s)
+    "%02dh%02dm%02ds" % [s / 3600, s / 60 % 60, s % 60]
+  end
+
   # Extract pod info from json data. The data is expected to be from the kubectl
   # command and conform to kubernetes' datatype structures.
   #
   # @param json_data [#to_h]
   #   the pod data returned from 'kubectl get pod abc-123'
+  # @param ns_prefix [#to_s]
+  #   the namespace prefix so that namespaces can be converted back to usernames
   # @return [#to_h]
   #   the hash of info expected from adapters
-  def pod_info_from_json(json_data)
+  def pod_info_from_json(json_data, ns_prefix: nil)
     {
       id: json_data.dig(:metadata, :name).to_s,
       job_name: name_from_metadata(json_data.dig(:metadata)),
       status: pod_status_from_json(json_data),
-      job_owner: json_data.dig(:metadata, :namespace).to_s,
+      job_owner: job_owner_from_json(json_data, ns_prefix),
       submission_time: submission_time(json_data),
       dispatch_time: dispatch_time(json_data),
       wallclock_time: wallclock_time(json_data),
-      native: {
-        host: get_host(json_data.dig(:status, :hostIP))
-      },
+      ood_connection_info: { host: get_host(json_data.dig(:status, :hostIP)) },
       procs: procs_from_json(json_data)
     }
   rescue NoMethodError
@@ -162,39 +180,24 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   def service_info_from_json(json_data)
     # all we need is the port - .spec.ports[0].nodePort
     ports = json_data.dig(:spec, :ports)
-    {
-      native:
-        {
-          port: ports[0].dig(:nodePort)
-        }
-    }
+    { ood_connection_info: { port: ports[0].dig(:nodePort) } }
   rescue
-    empty_native
+    {}
   end
 
   def secret_info_from_json(json_data)
     raw = json_data.dig(:data, :password)
-    {
-      native:
-        {
-          password: Base64.decode64(raw)
-        }
-    }
+    { ood_connection_info: { password: Base64.decode64(raw) } }
   rescue
-    empty_native
-  end
-
-  def empty_native
-    {
-      native: {}
-    }
+    {}
   end
 
   def dispatch_time(json_data)
     status = pod_status_from_json(json_data)
-    return nil if status == 'undetermined'
+    container_statuses = json_data.dig(:status, :containerStatuses)
+    return nil if container_statuses.nil?
 
-    state_data = json_data.dig(:status, :containerStatuses)[0].dig(:state)
+    state_data = container_statuses[0].dig(:state)
     date_string = nil
 
     if status == 'completed'
@@ -208,9 +211,10 @@ class OodCore::Job::Adapters::Kubernetes::Helper
 
   def wallclock_time(json_data)
     status = pod_status_from_json(json_data)
-    return nil if status == 'undetermined'
+    container_statuses = json_data.dig(:status, :containerStatuses)
+    return nil if container_statuses.nil?
 
-    state_data = json_data.dig(:status, :containerStatuses)[0].dig(:state)
+    state_data = container_statuses[0].dig(:state)
     start_time = dispatch_time(json_data)
     return nil if start_time.nil?
 
@@ -250,20 +254,21 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   end
 
   def pod_status_from_json(json_data)
-    state = 'undetermined'
-    status = json_data.dig(:status)
-    container_statuses = status.dig(:containerStatuses)
-
-    if container_statuses.nil?
-      # if you're here, it means you're pending, probably unschedulable
-      return OodCore::Job::Status.new(state: state)
-    end
-
-    # only support 1 container/pod
-    json_state = container_statuses[0].dig(:state)
-    state = 'running' unless json_state.dig(:running).nil?
-    state = terminated_state(json_state) unless json_state.dig(:terminated).nil?
-    state = 'queued' unless json_state.dig(:waiting).nil?
+    phase = json_data.dig(:status, :phase)
+    state = case phase
+            when "Running"
+              "running"
+            when "Pending"
+              "queued"
+            when "Failed"
+              "suspended"
+            when "Succeeded"
+              "completed"
+            when "Unknown"
+              "undetermined"
+            else
+              "undetermined"
+            end
 
     OodCore::Job::Status.new(state: state)
   end
@@ -295,4 +300,9 @@ class OodCore::Job::Adapters::Kubernetes::Helper
       cpu
     end
   end
-end
+
+  def job_owner_from_json(json_data = {}, ns_prefix = nil)
+    namespace = json_data.dig(:metadata, :namespace).to_s
+    namespace.delete_prefix(ns_prefix.to_s)
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb

@@ -0,0 +1,9 @@
+# An object that describes a submitted kubernetes job with extended information
+class OodCore::Job::Adapters::Kubernetes::K8sJobInfo < OodCore::Job::Info
+  attr_reader :ood_connection_info
+
+  def initialize(ood_connection_info: {}, **options)
+    super(options)
+    @ood_connection_info = ood_connection_info
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/resources.rb

@@ -1,22 +1,45 @@
 module OodCore::Job::Adapters::Kubernetes::Resources
 
   class ConfigMap
-    attr_accessor :name, :filename, :data
+    attr_accessor :name, :files
 
-    def initialize(name, filename, data)
+    def initialize(name, files)
       @name = name
-      @filename = filename
-      @data = data
+      @files = []
+      files.each do |f|
+        @files << ConfigMapFile.new(f)
+      end
+    end
+
+    def mounts?
+      @files.any? { |f| f.mount_path }
+    end
+
+    def init_mounts?
+      @files.any? { |f| f.init_mount_path }
+    end
+  end
+
+  class ConfigMapFile
+    attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
+
+    def initialize(data)
+      @filename = data[:filename]
+      @data = data[:data]
+      @mount_path = data[:mount_path]
+      @sub_path = data[:sub_path]
+      @init_mount_path = data[:init_mount_path]
+      @init_sub_path = data[:init_sub_path]
     end
   end
 
   class Container
     attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
-                  :restart_policy
+                  :restart_policy, :image_pull_policy, :image_pull_secret, :supplemental_groups
 
     def initialize(
-        name, image, command: [], port: nil, env: [], memory: "4Gi", cpu: "1",
-        working_dir: "", restart_policy: "Never"
+        name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
+        working_dir: "", restart_policy: "Never", image_pull_policy: nil, image_pull_secret: nil, supplemental_groups: []
       )
       raise ArgumentError, "containers need valid names and images" unless name && image
 
@@ -24,11 +47,14 @@ module OodCore::Job::Adapters::Kubernetes::Resources
       @image = image
       @command = command.nil? ? [] : command
       @port = port&.to_i
-      @env = env.nil? ? [] : env
+      @env = env.nil? ? {} : env
       @memory = memory.nil? ? "4Gi" : memory
       @cpu = cpu.nil? ? "1" : cpu
       @working_dir = working_dir.nil? ? "" : working_dir
       @restart_policy = restart_policy.nil? ? "Never" : restart_policy
+      @image_pull_policy = image_pull_policy.nil? ? "IfNotPresent" : image_pull_policy
+      @image_pull_secret = image_pull_secret
+      @supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
     end
 
     def ==(other)
@@ -40,9 +66,11 @@ module OodCore::Job::Adapters::Kubernetes::Resources
         memory == other.memory &&
         cpu == other.cpu &&
         working_dir == other.working_dir &&
-        restart_policy == other.restart_policy
+        restart_policy == other.restart_policy &&
+        image_pull_policy == other.image_pull_policy &&
+        image_pull_secret == other.image_pull_secret &&
+        supplemental_groups == other.supplemental_groups
     end
-
   end
 
   class PodSpec

data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb

@@ -7,100 +7,187 @@ metadata:
     job: <%= id %>
     app.kubernetes.io/name: <%= container.name %>
     app.kubernetes.io/managed-by: open-ondemand
+    <%- if !script.accounting_id.nil? && script.accounting_id != "" -%>
+    account: <%= script.accounting_id %>
+    <%- end -%>
+  annotations:
+    <%- unless script.wall_time.nil? -%>
+    pod.kubernetes.io/lifetime: <%= helper.seconds_to_duration(script.wall_time) %>
+    <%- end -%>
 spec:
   restartPolicy: <%= spec.container.restart_policy %>
   securityContext:
     runAsUser: <%= run_as_user %>
     runAsGroup: <%= run_as_group %>
+    runAsNonRoot: true
+    <%- if spec.container.supplemental_groups.empty? -%>
+    supplementalGroups: []
+    <%- else -%>
+    supplementalGroups:
+    <%- spec.container.supplemental_groups.each do |supplemental_group| -%>
+    - "<%= supplemental_group %>"
+    <%- end -%>
+    <%- end -%>
     fsGroup: <%= fs_group %>
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  <%- unless spec.container.image_pull_secret.nil? -%>
+  imagePullSecrets:
+  - name: <%= spec.container.image_pull_secret %>
+  <%- end -%>
   containers:
   - name: "<%= spec.container.name %>"
     image: <%= spec.container.image %>
-    imagePullPolicy: IfNotPresent
-    <% unless spec.container.working_dir.empty? %>
+    imagePullPolicy: <%= spec.container.image_pull_policy %>
+    <%- unless spec.container.working_dir.empty? -%>
     workingDir: "<%= spec.container.working_dir %>"
-    <% end %>
-    <% unless spec.container.env.empty? %>
+    <%- end -%>
     env:
-    <% spec.container.env.each do |env| %>
-    - name: <%= env[:name] %>
-      value: "<%= env[:value] %>"
-    <% end %> <%# for each env %>
-    <% end %> <%# unless env is nil %>
-    <% unless spec.container.command.empty? %>
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    <%- spec.container.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
+    <%- end # for each env -%>
+    <%- unless spec.container.command.empty? -%>
     command:
-    <% spec.container.command.each do |cmd| %>
+    <%- spec.container.command.each do |cmd| -%>
     - "<%= cmd %>"
-    <% end %> <%# for each command %>
-    <% end %> <%# unless command is nil %>
-    <% unless spec.container.port.nil? %>
+    <%- end # for each command -%>
+    <%- end # unless command is nil -%>
+    <%- unless spec.container.port.nil? -%>
     ports:
     - containerPort: <%= spec.container.port %>
-    <% end %> 
+    <%- end -%>
+    <%- if !all_mounts.empty? || (!configmap.nil? && configmap.mounts?) -%>
     volumeMounts:
-    <% unless configmap.nil? %>
+    <%- unless configmap.nil? -%>
+    <%- configmap.files.each do |file| -%>
+      <%- next if file.mount_path.nil? -%>
     - name: configmap-volume
-      mountPath:  <%= configmap_mount_path %>
-    <% end %>
-    <% all_mounts.each do |mount| %>
+      mountPath: <%= file.mount_path %>
+      <%- unless file.sub_path.nil? -%>
+      subPath: <%= file.sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
+    <%- end # unless configmap.nil? -%>
+    <%- all_mounts.each do |mount| -%>
     - name: <%= mount[:name] %>
       mountPath: <%= mount[:destination_path] %>
-    <% end %> <%# for each mount %>
+    <%- end # for each mount -%>
+    <%- end # configmap mounts? and all_mounts not empty -%>
     resources:
       limits:
         memory: "<%= spec.container.memory %>"
         cpu: "<%= spec.container.cpu %>"
+        <%- unless script.gpus_per_node.nil? -%>
+        <%= gpu_type %>: <%= script.gpus_per_node %>
+        <%- end -%>
       requests:
         memory: "<%= spec.container.memory %>"
         cpu: "<%= spec.container.cpu %>"
-  <% unless spec.init_containers.nil? %>
+        <%- unless script.gpus_per_node.nil? -%>
+        <%= gpu_type %>: <%= script.gpus_per_node %>
+        <%- end -%>
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - all
+      privileged: false
+  <%- unless spec.init_containers.empty? -%>
   initContainers:
-  <% spec.init_containers.each do |ctr| %>
+  <%- spec.init_containers.each do |ctr| -%>
   - name: "<%= ctr.name %>"
     image: "<%= ctr.image %>"
+    imagePullPolicy: <%= ctr.image_pull_policy %>
+    env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    <%- ctr.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
+    <%- end # for each env -%>
     command:
-    <% ctr.command.each do |cmd| %>
+    <%- ctr.command.each do |cmd| -%>
     - "<%= cmd %>"
-    <% end %> <%# command loop  %>
+    <%- end # command loop -%>
+    <%- if !all_mounts.empty? || (!configmap.nil? && configmap.init_mounts?) -%>
     volumeMounts:
-    <% unless configmap.nil? %>
+    <%- unless configmap.nil? -%>
+    <%- configmap.files.each do |file| -%>
+    <%- next if file.init_mount_path.nil? -%>
     - name: configmap-volume
-      mountPath:  <%= configmap_mount_path %>
-    <% end %>
-    <% all_mounts.each do |mount| %>
+      mountPath: <%= file.init_mount_path %>
+      <%- unless file.init_sub_path.nil? -%>
+      subPath: <%= file.init_sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
+    <%- end # unless configmap.nil? -%>
+    <%- all_mounts.each do |mount| -%>
     - name: <%= mount[:name] %>
       mountPath: <%= mount[:destination_path] %>
-    <% end %> <%# for each mount %>
-  <% end %> <%# init container loop  %>
-  <% end %> <%# if init containers  %>
-  <% unless configmap.nil? || all_mounts.empty? %>
+    <%- end # for each mount -%>
+    <%- end # if config_map init mounts and all_mounts not empty -%>
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - all
+      privileged: false
+  <%- end # init container loop -%>
+  <%- end # if init containers -%>
+  <%- unless (configmap.to_s.empty? && all_mounts.empty?) -%>
   volumes:
-  <% end %> <%# configmap.nil? || all_mounts.empty? %>
-  <% unless configmap.nil? %>
+  <%- unless configmap.to_s.empty? -%>
   - name: configmap-volume
     configMap:
       name: <%= configmap_name(id) %>
-  <% end %>
-  <% all_mounts.each do |mount| %>
-  <% if mount[:type] == 'nfs' %>
+  <%- end -%>
+  <%- all_mounts.each do |mount| -%>
+  <%- if mount[:type] == 'nfs' -%>
   - name: <%= mount[:name] %>
     nfs:
       server: <%= mount[:host] %>
       path: <%= mount[:path] %>
-  <% elsif mount[:type] == 'host' %>
+  <%- elsif mount[:type] == 'host' -%>
   - name: <%= mount[:name] %>
     hostPath:
       path: <%= mount[:path] %>
       type: <%= mount[:host_type] %>
-  <% end %> <%# if mount is [host,nfs] %>
-  <% end %> <%# for each mount %>
+  <%- end # if mount is [host,nfs] -%>
+  <%- end # for each mount -%>
+  <%- end # (configmap.to_s.empty? || all_mounts.empty?) -%>
+  <%- unless node_selector.empty? -%>
+  nodeSelector:
+  <%- node_selector.each_pair do |key, value| -%>
+    <%= key %>: "<%= value %>"
+  <%- end # node_selector.each_pair -%>
+  <%- end #unless node_selector.empty? -%>
 ---
-<% unless spec.container.port.nil? %>
+<%- unless spec.container.port.nil? -%>
 apiVersion: v1
 kind: Service
 metadata:
   name: <%= service_name(id) %>
   namespace: <%= namespace %>
+  labels:
+    job: <%= id %>
+    app.kubernetes.io/name: <%= container.name %>
+    app.kubernetes.io/managed-by: open-ondemand
 spec:
   selector:
     job: <%= id %>
@@ -109,15 +196,22 @@ spec:
     port: 80
     targetPort: <%= spec.container.port %>
   type: NodePort
-<% end %> <%# end for service %>
+<%- end # end for service -%>
+<%- unless configmap.nil? -%>
 ---
-<% unless configmap.nil? %>
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: <%= configmap_name(id) %>
   namespace: <%= namespace %>
+  labels:
+    job: <%= id %>
+    app.kubernetes.io/name: <%= container.name %>
+    app.kubernetes.io/managed-by: open-ondemand
 data:
-  <%= configmap.filename %>: |
-    <% config_data_lines(configmap.data).each do |line| %><%= line %><% end %>
-<% end %> <%# end for configmap %>
+<%- configmap.files.each do |file| -%>
+  <%- next if file.data.nil? || file.filename.nil? -%>
+  <%= file.filename %>: |
+    <% config_data_lines(file.data).each do |line| %><%= line %><% end %>
+<%- end # end for configmap files -%>
+<%- end # end configmap.nil? %>