ood_core 0.12.0 → 0.16.0

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -0,0 +1,372 @@
+require "ood_core/refinements/hash_extensions"
+require "json"
+
+class OodCore::Job::Adapters::Kubernetes::Batch
+
+  require_relative "helper"
+  require_relative "k8s_job_info"
+
+  using OodCore::Refinements::HashExtensions
+
+  class Error < StandardError; end
+  class NotFoundError < StandardError; end
+
+  attr_reader :config_file, :bin, :cluster, :mounts
+  attr_reader :all_namespaces, :using_context, :helper
+  attr_reader :username_prefix, :namespace_prefix
+
+  def initialize(options = {})
+    options = options.to_h.symbolize_keys
+
+    @config_file = options.fetch(:config_file, default_config_file)
+    @bin = options.fetch(:bin, '/usr/bin/kubectl')
+    @cluster = options.fetch(:cluster, 'open-ondemand')
+    @mounts = options.fetch(:mounts, []).map { |m| m.to_h.symbolize_keys }
+    @all_namespaces = options.fetch(:all_namespaces, false)
+    @username_prefix = options.fetch(:username_prefix, nil)
+    @namespace_prefix = options.fetch(:namespace_prefix, '')
+
+    @using_context = false
+    @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
+
+    begin
+      make_kubectl_config(options)
+    rescue
+      # FIXME could use a log here
+      # means you couldn't 'kubectl set config'
+    end
+  end
+
+  def resource_file(resource_type = 'pod')
+    File.dirname(__FILE__) + "/templates/#{resource_type}.yml.erb"
+  end
+
+  def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+    raise ArgumentError, 'Must specify the script' if script.nil?
+
+    resource_yml, id = generate_id_yml(script)
+    call("#{formatted_ns_cmd} create -f -", stdin: resource_yml)
+
+    id
+  end
+
+  def generate_id(name)
+    # 2_821_109_907_456 = 36**8
+    name.downcase.tr(' ', '-') + '-' + rand(2_821_109_907_456).to_s(36)
+  end
+
+  def info_all(attrs: nil)
+    cmd = if all_namespaces
+            "#{base_cmd} get pods -o json --all-namespaces"
+          else
+            "#{namespaced_cmd} get pods -o json"
+          end
+
+    output = call(cmd)
+    all_pods_to_info(output)
+  end
+
+  def info_where_owner(owner, attrs: nil)
+    owner = Array.wrap(owner).map(&:to_s)
+
+    # must at least have job_owner to filter by job_owner
+    attrs = Array.wrap(attrs) | [:job_owner] unless attrs.nil?
+
+    info_all(attrs: attrs).select { |info| owner.include? info.job_owner }
+  end
+
+  def info_all_each(attrs: nil)
+    return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+    info_all(attrs: attrs).each do |job|
+      yield job
+    end
+  end
+
+  def info_where_owner_each(owner, attrs: nil)
+    return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+    info_where_owner(owner, attrs: attrs).each do |job|
+      yield job
+    end
+  end
+
+  def info(id)
+    pod_json = safe_call('get', 'pod', id)
+    return OodCore::Job::Info.new({ id: id, status: 'completed' }) if pod_json.empty?
+
+    service_json = safe_call('get', 'service', service_name(id))
+    secret_json = safe_call('get', 'secret', secret_name(id))
+
+    helper.info_from_json(pod_json: pod_json, service_json: service_json, secret_json: secret_json)
+  end
+
+  def status(id)
+    info(id).status
+  end
+
+  def delete(id)
+    safe_call("delete", "pod", id)
+    safe_call("delete", "service", service_name(id))
+    safe_call("delete", "secret", secret_name(id))
+    safe_call("delete", "configmap", configmap_name(id))
+  end
+
+  private
+
+  def safe_call(verb, resource, id)
+    begin
+      case verb.to_s
+      when "get"
+        call_json_output('get', resource, id)
+      when "delete"
+        call("#{namespaced_cmd} delete #{resource} #{id}")
+      end
+    rescue NotFoundError
+      {}
+    end
+  end
+
+  # helper to help format multi-line yaml data from the submit.yml into 
+  # mutli-line yaml in the pod.yml.erb
+  def config_data_lines(data)
+    output = []
+    first = true
+
+    data.to_s.each_line do |line|
+      output.append(first ? line : line.prepend("    "))
+      first = false
+    end
+
+    output
+  end
+
+  def username
+    @username ||= Etc.getlogin
+  end
+
+  def k8s_username
+    username_prefix.nil? ? username : "#{username_prefix}-#{username}"
+  end
+
+  def user
+    @user ||= Etc.getpwnam(username)
+  end
+
+  def home_dir
+    user.dir
+  end
+
+  def run_as_user
+    user.uid
+  end
+
+  def run_as_group
+    user.gid
+  end
+
+  def fs_group
+    run_as_group
+  end
+
+  def group
+    Etc.getgrgid(run_as_group).name
+  end
+
+  def default_env
+    {
+      USER: username,
+      UID: run_as_user,
+      HOME: home_dir,
+      GROUP: group,
+      GID: run_as_group,
+    }
+  end
+
+  # helper to template resource yml you're going to submit and
+  # create an id.
+  def generate_id_yml(script)
+    native_data = script.native
+    container = helper.container_from_native(native_data[:container], default_env)
+    id = generate_id(container.name)
+    configmap = helper.configmap_from_native(native_data, id)
+    init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
+    spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
+    all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
+
+    template = ERB.new(File.read(resource_file), nil, '-')
+
+    [template.result(binding), id]
+  end
+
+  # helper to call kubectl and get json data back.
+  # verb, resrouce and id are the kubernetes parlance terms.
+  # example: 'kubectl get pod my-pod-id' is verb=get, resource=pod
+  # and  id=my-pod-id
+  def call_json_output(verb, resource, id, stdin: nil)
+    cmd = "#{formatted_ns_cmd} #{verb} #{resource} #{id}"
+    data = call(cmd, stdin: stdin)
+    data = data.empty? ? '{}' : data
+    json_data = JSON.parse(data, symbolize_names: true)
+
+    json_data
+  end
+
+  def service_name(id)
+    helper.service_name(id)
+  end
+
+  def secret_name(id)
+    helper.secret_name(id)
+  end
+
+  def configmap_name(id)
+    helper.configmap_name(id)
+  end
+
+  def namespace
+    "#{namespace_prefix}#{username}"
+  end
+
+  def context
+    cluster
+  end
+
+  def default_config_file
+    (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
+  end
+
+  def default_auth
+    {
+      type: 'managaged'
+    }.symbolize_keys
+  end
+
+  def default_server
+    {
+      endpoint: 'https://localhost:8080',
+      cert_authority_file: nil
+    }.symbolize_keys
+  end
+
+  def formatted_ns_cmd
+    "#{namespaced_cmd} -o json"
+  end
+
+  def namespaced_cmd
+    "#{base_cmd} --namespace=#{namespace}"
+  end
+
+  def base_cmd
+    base = "#{bin} --kubeconfig=#{config_file}"
+    base << " --context=#{context}" if using_context
+    base
+  end
+
+  def all_pods_to_info(data)
+    json_data = JSON.parse(data, symbolize_names: true)
+    pods = json_data.dig(:items)
+
+    info_array = []
+    pods.each do |pod|
+      info = pod_info_from_json(pod)
+      info_array.push(info) unless info.nil?
+    end
+
+    info_array
+  rescue JSON::ParserError
+    # 'no resources in <namespace>' throws parse error
+    []
+  end
+
+  def pod_info_from_json(pod)
+    hash = helper.pod_info_from_json(pod)
+    K8sJobInfo.new(hash)
+  rescue Helper::K8sDataError
+    # FIXME: silently eating error, could probably use a logger
+    nil
+  end
+
+  def make_kubectl_config(config)
+    set_cluster(config.fetch(:server, default_server).to_h.symbolize_keys)
+    configure_auth(config.fetch(:auth, default_auth).to_h.symbolize_keys)
+  end
+
+  def configure_auth(auth)
+    type = auth.fetch(:type)
+    return if managed?(type)
+
+    case type
+    when 'gke'
+      set_gke_config(auth)
+    when 'oidc'
+      set_context
+    end
+  end
+
+  def use_context
+    @using_context = true
+  end
+
+  def managed?(type)
+    if type.nil?
+      true # maybe should be false?
+    else
+      type.to_s == 'managed'
+    end
+  end
+
+  def set_gke_config(auth)
+    cred_file = auth.fetch(:svc_acct_file)
+
+    cmd = "gcloud auth activate-service-account --key-file=#{cred_file}"
+    call(cmd)
+
+    set_gke_credentials(auth)
+  end
+
+  def set_gke_credentials(auth)
+
+    zone = auth.fetch(:zone, nil)
+    region = auth.fetch(:region, nil)
+
+    locale = ''
+    locale = "--zone=#{zone}" unless zone.nil?
+    locale = "--region=#{region}" unless region.nil?
+
+    # gke cluster name can probably can differ from what ood calls the cluster
+    cmd = "gcloud container clusters get-credentials #{locale} #{cluster}"
+    env = { 'KUBECONFIG' => config_file }
+    call(cmd, env)
+  end
+
+  def set_context
+    cmd = "#{base_cmd} config set-context #{cluster}"
+    cmd << " --cluster=#{cluster} --namespace=#{namespace}"
+    cmd << " --user=#{k8s_username}"
+
+    call(cmd)
+    use_context
+  end
+
+  def set_cluster(config)
+    server = config.fetch(:endpoint)
+    cert = config.fetch(:cert_authority_file, nil)
+
+    cmd = "#{base_cmd} config set-cluster #{cluster}"
+    cmd << " --server=#{server}"
+    cmd << " --certificate-authority=#{cert}" unless cert.nil?
+
+    call(cmd)
+  end
+
+  def call(cmd = '', env: {}, stdin: nil)
+    o, e, s = Open3.capture3(env, cmd, stdin_data: stdin.to_s)
+    s.success? ? o : interpret_and_raise(e)
+  end
+
+  def interpret_and_raise(stderr)
+    raise NotFoundError, stderr if /^Error from server \(NotFound\):/.match(stderr)
+    raise(Error, stderr)
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -0,0 +1,299 @@
+class OodCore::Job::Adapters::Kubernetes::Helper
+
+  require_relative 'resources'
+  require_relative 'k8s_job_info'
+  require 'resolv'
+  require 'base64'
+  require 'active_support/core_ext/hash'
+
+  class K8sDataError < StandardError; end
+
+  # Extract info from json data. The data is expected to be from the kubectl
+  # command and conform to kubernetes' datatype structures.
+  #
+  # Returns K8sJobInfo in the in lieu of writing a connection.yml
+  #
+  # @param pod_json [#to_h]
+  #   the pod data returned from 'kubectl get pod abc-123'
+  # @param service_json [#to_h]
+  #   the service data returned from 'kubectl get service abc-123-service'
+  # @param secret_json [#to_h]
+  #   the secret data returned from 'kubectl get secret abc-123-secret'
+  # @param ns_prefix [#to_s]
+  #   the namespace prefix so that namespaces can be converted back to usernames
+  # @return [OodCore::Job::Adapters::Kubernetes::K8sJobInfo]
+  def info_from_json(pod_json: nil, service_json: nil, secret_json: nil, ns_prefix: nil)
+    pod_hash = pod_info_from_json(pod_json, ns_prefix: ns_prefix)
+    service_hash = service_info_from_json(service_json)
+    secret_hash = secret_info_from_json(secret_json)
+
+    pod_hash.deep_merge!(service_hash)
+    pod_hash.deep_merge!(secret_hash)
+    OodCore::Job::Adapters::Kubernetes::K8sJobInfo.new(pod_hash)
+  rescue NoMethodError
+    raise K8sDataError, "unable to read data correctly from json"
+  end
+
+  # Turn a container hash into a Kubernetes::Resources::Container
+  #
+  # @param container [#to_h]
+  #   the input container hash
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
+  # @return  [OodCore::Job::Adapters::Kubernetes::Resources::Container]
+  def container_from_native(container, default_env)
+    env = container.fetch(:env, {}).to_h.symbolize_keys
+    OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
+      container[:name],
+      container[:image],
+      command: parse_command(container[:command]),
+      port: container[:port],
+      env: default_env.merge(env),
+      memory: container[:memory],
+      cpu: container[:cpu],
+      working_dir: container[:working_dir],
+      restart_policy: container[:restart_policy],
+      image_pull_secret: container[:image_pull_secret]
+    )
+  end
+
+  # Parse a command string given from a user and return an array.
+  # If given an array, the input is simply returned back.
+  #
+  # @param cmd [#to_s]
+  #   the command to parse
+  # @return [Array<#to_s>]
+  #   the command parsed into an array of arguements
+  def parse_command(cmd)
+    if cmd&.is_a?(Array)
+      cmd
+    else
+      Shellwords.split(cmd.to_s)
+    end
+  end
+
+  # Turn a configmap hash into a Kubernetes::Resources::ConfigMap
+  # that can be used in templates. Needs an id so that the resulting
+  # configmap has a known name.
+  #
+  # @param native [#to_h]
+  #   the input configmap hash
+  # @param id [#to_s]
+  #   the id to use for giving the configmap a name
+  # @return  [OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap]
+  def configmap_from_native(native, id)
+    configmap = native.fetch(:configmap, nil)
+    return nil if configmap.nil?
+
+    OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
+      configmap_name(id),
+      (configmap[:files] || [])
+    )
+  end
+
+  # parse initialization containers from native data
+  #
+  # @param native_data [#to_h]
+  #   the native data to parse. Expected key init_ctrs and for that
+  #   key to be an array of hashes.
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
+  # @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
+  #   the array of init containers
+  def init_ctrs_from_native(ctrs, default_env)
+    init_ctrs = []
+
+    ctrs&.each do |ctr_raw|
+      ctr = container_from_native(ctr_raw, default_env)
+      init_ctrs.push(ctr)
+    end
+
+    init_ctrs
+  end
+
+  def service_name(id)
+    id + '-service'
+  end
+
+  def secret_name(id)
+    id + '-secret'
+  end
+
+  def configmap_name(id)
+    id + '-configmap'
+  end
+
+  def seconds_to_duration(s)
+    "%02dh%02dm%02ds" % [s / 3600, s / 60 % 60, s % 60]
+  end
+
+  # Extract pod info from json data. The data is expected to be from the kubectl
+  # command and conform to kubernetes' datatype structures.
+  #
+  # @param json_data [#to_h]
+  #   the pod data returned from 'kubectl get pod abc-123'
+  # @param ns_prefix [#to_s]
+  #   the namespace prefix so that namespaces can be converted back to usernames
+  # @return [#to_h]
+  #   the hash of info expected from adapters
+  def pod_info_from_json(json_data, ns_prefix: nil)
+    {
+      id: json_data.dig(:metadata, :name).to_s,
+      job_name: name_from_metadata(json_data.dig(:metadata)),
+      status: pod_status_from_json(json_data),
+      job_owner: job_owner_from_json(json_data, ns_prefix),
+      submission_time: submission_time(json_data),
+      dispatch_time: dispatch_time(json_data),
+      wallclock_time: wallclock_time(json_data),
+      ood_connection_info: { host: get_host(json_data.dig(:status, :hostIP)) },
+      procs: procs_from_json(json_data)
+    }
+  rescue NoMethodError
+    # gotta raise an error because Info.new will throw an error if id is undefined
+    raise K8sDataError, "unable to read data correctly from json"
+  end
+
+  private
+
+  def get_host(ip)
+    Resolv.getname(ip)
+  rescue Resolv::ResolvError
+    ip
+  end
+
+  def name_from_metadata(metadata)
+    name = metadata.dig(:labels, :'app.kubernetes.io/name')
+    name = metadata.dig(:labels, :'k8s-app') if name.nil?
+    name = metadata.dig(:name) if name.nil? # pod-id but better than nil?
+    name
+  end
+
+  def service_info_from_json(json_data)
+    # all we need is the port - .spec.ports[0].nodePort
+    ports = json_data.dig(:spec, :ports)
+    { ood_connection_info: { port: ports[0].dig(:nodePort) } }
+  rescue
+    {}
+  end
+
+  def secret_info_from_json(json_data)
+    raw = json_data.dig(:data, :password)
+    { ood_connection_info: { password: Base64.decode64(raw) } }
+  rescue
+    {}
+  end
+
+  def dispatch_time(json_data)
+    status = pod_status_from_json(json_data)
+    container_statuses = json_data.dig(:status, :containerStatuses)
+    return nil if container_statuses.nil?
+
+    state_data = container_statuses[0].dig(:state)
+    date_string = nil
+
+    if status == 'completed'
+      date_string = state_data.dig(:terminated, :startedAt)
+    elsif status == 'running'
+      date_string = state_data.dig(:running, :startedAt)
+    end
+
+    date_string.nil? ? nil : DateTime.parse(date_string).to_time.to_i
+  end
+
+  def wallclock_time(json_data)
+    status = pod_status_from_json(json_data)
+    container_statuses = json_data.dig(:status, :containerStatuses)
+    return nil if container_statuses.nil?
+
+    state_data = container_statuses[0].dig(:state)
+    start_time = dispatch_time(json_data)
+    return nil if start_time.nil?
+
+    et = end_time(status, state_data)
+
+    et.nil? ? nil : et - start_time
+  end
+
+  def end_time(status, state_data)
+    if status == 'completed'
+      end_time_string = state_data.dig(:terminated, :finishedAt)
+      et = DateTime.parse(end_time_string).to_time.to_i
+    elsif status == 'running'
+      et = DateTime.now.to_time.to_i
+    else
+      et = nil
+    end
+
+    et
+  end
+
+  def submission_time(json_data)
+    status = json_data.dig(:status)
+    start = status.dig(:startTime)
+
+    if start.nil?
+      # the pod is in some pending state limbo
+      conditions = status.dig(:conditions)
+      # best guess to start time is just the first condition's
+      # transition time
+      str = conditions[0].dig(:lastTransitionTime)
+    else
+      str = start
+    end
+
+    DateTime.parse(str).to_time.to_i
+  end
+
+  def pod_status_from_json(json_data)
+    phase = json_data.dig(:status, :phase)
+    state = case phase
+            when "Running"
+              "running"
+            when "Pending"
+              "queued"
+            when "Failed"
+              "suspended"
+            when "Succeeded"
+              "completed"
+            when "Unknown"
+              "undetermined"
+            else
+              "undetermined"
+            end
+
+    OodCore::Job::Status.new(state: state)
+  end
+
+  def terminated_state(status)
+    reason = status.dig(:terminated, :reason)
+    if reason == 'Error'
+      'suspended'
+    else
+      'completed'
+    end
+  end
+
+  def procs_from_json(json_data)
+    containers = json_data.dig(:spec, :containers)
+    resources = containers[0].dig(:resources)
+
+    cpu = resources.dig(:limits, :cpu)
+    millicores_rex = /(\d+)m/
+
+    # ok to return string bc nil.to_i == 0 and we'd rather return
+    # nil (undefined) than 0 which is confusing.
+    if millicores_rex.match?(cpu)
+      millicores =  millicores_rex.match(cpu)[1].to_i
+
+      # have to return at least 1 bc 200m could be 0
+      ((millicores + 1000) / 1000).to_s
+    else
+      cpu
+    end
+  end
+
+  def job_owner_from_json(json_data = {}, ns_prefix = nil)
+    namespace = json_data.dig(:metadata, :namespace).to_s
+    namespace.delete_prefix(ns_prefix.to_s)
+  end
+end